diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9688522e..ed6cbb10 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -65,6 +65,7 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-util = { module = "org.eclipse.edc:util", version.ref = "edc" } edc-vault-azure = { module = "org.eclipse.edc.azure:vault-azure", version.ref = "edc" } +edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" } edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } jakartaJson = { module = "org.glassfish:jakarta.json", version.ref = "jakarta-json" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 90c0d9cb..56720d6f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,6 +52,9 @@ include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") include(":policy:policy-01-policy-enforcement:policy-functions") +include(":transfer:transfer-06-file-transfer-docker:docker-transfer-consumer") +include(":transfer:transfer-06-file-transfer-docker:docker-transfer-provider") + // modules for code samples ------------------------------------------------------------------------ include(":advanced:advanced-02-custom-runtime") diff --git a/transfer/transfer-06-file-transfer-docker/README.md b/transfer/transfer-06-file-transfer-docker/README.md new file mode 100644 index 00000000..b68f1492 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/README.md @@ -0,0 +1,172 @@ +# Improve the file transfer + +The previous transfer from the previous chapter was a transfer in the cloud. +So that this can also be done without cloud accounts, a transfer between two docker images is carried out in this chapter. +Instead of a transfer from Azure Storage to an AWS S3 Bucket, we will: + +- read the source from Azurite, +- put the destination file into Minio. + + +## Start the docker-compose file + +```bash +docker-compose -f transfer/transfer-06-file-transfer-docker/resources/docker-compose.yml up -d +``` + +You should see this: +```sh + ✔ Container azurite Started + ✔ Container minio Started +``` + +## Create bucket in minio + +Go to http://localhost:9001 and login with the credentials which you can find in the [docker-compose](resources/docker-compose.yaml) file (line 23-24), then go to 'Buckets' and create a bucket with the name “src-bucket”. + +## Upload file to azurite +Before we upload the file, you have to install Amazon CLI. After that, you have to create a blob storage: + +```bash +conn_str="DefaultEndpointsProtocol=http;AccountName=provider;AccountKey=password;BlobEndpoint=http://127.0.0.1:10000/provider;" +az storage container create --name src-container --connection-string $conn_str +``` + +If the storage is created successfully, you will get something like this: +```json +{ + "created": true +} +``` + +Upload the file to the blob storage: + +```bash +az storage blob upload -f ./transfer/transfer-06-file-transfer-docker/resources/test-document.txt --container-name src-container --name test-document.txt --connection-string $conn_str +``` + +You can run the following command to check if the file was added successfully + +```bash +az storage blob list --container-name src-container --connection-string "DefaultEndpointsProtocol=http;AccountName=provider;AccountKey=password;BlobEndpoint=http://127.0.0.1:10000/provider;" --query "[].{name:name}" --output table +``` + +You should see the test-document.txt file. + +```sh +Name +-------------------------- +test-document.txt +``` + + +## Start the vault +Now that we have both uploaded the file on azurite and created the bucket on minio, we can install hashicorp-vault. After installation, you can start the vault server: + +```bash +vault server -dev +``` +Now on another terminal window: +```bash +export VAULT_ADDR='http://127.0.0.1:8200' +vault kv put secret/accessKeyId content=consumer +vault kv put secret/secretAccessKey content=password +vault kv put secret/provider-key content=password +``` + + +## Update connector config + +_Do the following for both the consumer's and the provider's `config.properties`!_ + +Let's modify the following config values to the connector configuration `config.properties` and insert the root-token that vault issued when we started the server: + +```properties +edc.vault.hashicorp.token= +``` + +## Bringing it all together + +### 1. Boot connectors + +```bash +./gradlew clean build +java -Dedc.fs.config=transfer/transfer-06-file-transfer-docker/docker-transfer-provider/config.properties -jar transfer/transfer-06-file-transfer-docker/docker-transfer-provider/build/libs/provider.jar +# in another terminal window: +java -Dedc.fs.config=transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/config.properties -jar transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/build/libs/consumer.jar +``` + +### 2. Create a Asset on the provider + +```bash +curl -d @transfer/transfer-06-file-transfer-docker/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:19193/management/v3/assets \ + -s | jq +``` + +### 3. Create a Policy on the provider + +```bash +curl -d @transfer/transfer-06-file-transfer-docker/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:19193/management/v3/policydefinitions \ + -s | jq +``` + +### 4. Create a contract definition on Provider + +```bash +curl -d @transfer/transfer-06-file-transfer-docker/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:19193/management/v3/contractdefinitions \ + -s | jq +``` + +### 5. Fetch catalog on consumer side + +```bash +curl -X POST "http://localhost:29193/management/v3/catalog/request" \ + -H 'Content-Type: application/json' \ + -d @transfer/transfer-06-file-transfer-docker/resources/fetch-catalog.json -s | jq +``` + +Please replace the {{contract-offer-id}} placeholder in the [negotiate-contract.json](resources/negotiate-contract.json) file with the contract offer id you found in the catalog at the path dcat:dataset.odrl:hasPolicy.@id. + +### 6. Negotiate a contract + +```bash +curl -d @transfer/transfer-06-file-transfer-docker/resources/negotiate-contract.json \ + -X POST -H 'content-type: application/json' http://localhost:29193/management/v3/contractnegotiations \ + -s | jq +``` + +We can now use the UUID to check the current status of the negotiation using an endpoint on the consumer side. + +### 7. Getting the contract Agreement id + +```bash +curl -X GET "http://localhost:29193/management/v3/contractnegotiations/{{contract-negotiation-id}}" \ + --header 'Content-Type: application/json' \ + -s | jq +``` + +Please replace the {{contract-agreement-id}} placeholder in the [start-transfer.json](resources/start-transfer.json) file with the contractAgreementId from the previous response. + +### 8. Start the transfer + +```bash +curl -X POST "http://localhost:29193/management/v3/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-06-file-transfer-docker/resources/start-transfer.json \ + -s | jq +``` + +With the given UUID, we can check the transfer process. + +### 9. Check the transfer status + +```bash +curl http://localhost:29193/management/v3/transferprocesses/ -s | jq +``` + +--- + +[Previous Chapter](../transfer-04-open-telemetry/README.md) diff --git a/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/build.gradle.kts b/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/build.gradle.kts new file mode 100644 index 00000000..a762135a --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/build.gradle.kts @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2020, 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * Fraunhofer Institute for Software and Systems Engineering - added dependencies + * ZF Friedrichshafen AG - add dependency + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(libs.edc.control.api.configuration) + implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) + implementation(libs.edc.control.plane.core) + implementation(libs.edc.dsp) + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) + implementation(libs.edc.management.api) + implementation(libs.edc.transfer.data.plane.signaling) + implementation(libs.edc.transfer.pull.http.receiver) + implementation(libs.edc.validator.data.address.http.data) + + implementation(libs.edc.edr.cache.api) + implementation(libs.edc.edr.store.core) + implementation(libs.edc.edr.store.receiver) + + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + + implementation(libs.edc.data.plane.self.registration) + implementation(libs.edc.data.plane.control.api) + implementation(libs.edc.data.plane.public.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.vault.hashicorp) + +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("consumer.jar") +} diff --git a/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/config.properties b/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/config.properties new file mode 100644 index 00000000..79593339 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/docker-transfer-consumer/config.properties @@ -0,0 +1,21 @@ +edc.participant.id=consumer +edc.dsp.callback.address=http://localhost:29194/protocol +web.http.port=29191 +web.http.path=/api +web.http.management.port=29193 +web.http.management.path=/management +web.http.protocol.port=29194 +web.http.protocol.path=/protocol +edc.transfer.proxy.token.signer.privatekey.alias=private-key +edc.transfer.proxy.token.verifier.publickey.alias=public-key +web.http.public.port=29291 +web.http.public.path=/public +web.http.control.port=29192 +web.http.control.path=/control + +edc.converter.usefilesystem=false + +edc.vault.hashicorp.url=http://127.0.0.1:8200 +edc.vault.hashicorp.token= +edc.vault.hashicorp.api.secret.path=/v1/secret +edc.vault.hashicorp.health.check.enabled=false diff --git a/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/build.gradle.kts b/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/build.gradle.kts new file mode 100644 index 00000000..0f41a3e5 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/build.gradle.kts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020, 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * Fraunhofer Institute for Software and Systems Engineering - added dependencies + * ZF Friedrichshafen AG - add dependency + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + + implementation(libs.edc.control.api.configuration) + implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) + implementation(libs.edc.control.plane.core) + implementation(libs.edc.dsp) + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) + implementation(libs.edc.management.api) + implementation(libs.edc.transfer.data.plane.signaling) + implementation(libs.edc.transfer.pull.http.receiver) + implementation(libs.edc.validator.data.address.http.data) + + implementation(libs.edc.edr.cache.api) + implementation(libs.edc.edr.store.core) + implementation(libs.edc.edr.store.receiver) + + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + + implementation(libs.edc.data.plane.self.registration) + implementation(libs.edc.data.plane.control.api) + implementation(libs.edc.data.plane.public.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.data.plane.aws.s3) + implementation(libs.edc.data.plane.azure.storage) + + implementation(libs.edc.vault.hashicorp) + +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("provider.jar") +} diff --git a/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/config.properties b/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/config.properties new file mode 100644 index 00000000..02c4bb81 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/docker-transfer-provider/config.properties @@ -0,0 +1,30 @@ +edc.participant.id=provider +edc.dsp.callback.address=http://localhost:19194/protocol +web.http.port=19191 +web.http.path=/api +web.http.management.port=19193 +web.http.management.path=/management +web.http.protocol.port=19194 +web.http.protocol.path=/protocol +edc.transfer.proxy.token.signer.privatekey.alias=private-key +edc.transfer.proxy.token.verifier.publickey.alias=public-key +web.http.public.port=19291 +web.http.public.path=/public +web.http.control.port=19192 +web.http.control.path=/control +edc.dataplane.api.public.baseurl=http://localhost:19291/public + +edc.converter.usefilesystem=false + +edc.vault.hashicorp.url=http://127.0.0.1:8200 +edc.vault.hashicorp.token= +edc.vault.hashicorp.api.secret.path=/v1/secret +edc.vault.hashicorp.health.check.enabled=false + +#azurite endpoint +edc.blobstore.endpoint.template=http://127.0.0.1:10000/%s + +#minio connection +edc.aws.endpoint.override=http://localhost:9000 +edc.aws.access.key=accessKeyId +edc.aws.secret.access.key=secretAccessKey diff --git a/transfer/transfer-06-file-transfer-docker/resources/create-asset.json b/transfer/transfer-06-file-transfer-docker/resources/create-asset.json new file mode 100644 index 00000000..15ca1039 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/create-asset.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "assetId", + "properties": { + "name": "provider-test-document", + "description": "This is a test file", + "contenttype": "text/plain", + "version": "1.0" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "AzureStorage", + "account": "provider", + "keyName": "provider-key", + "container": "src-container", + "blobName": "test-document.txt" + } +} diff --git a/transfer/transfer-06-file-transfer-docker/resources/create-contract-definition.json b/transfer/transfer-06-file-transfer-docker/resources/create-contract-definition.json new file mode 100644 index 00000000..5ee5c67f --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/create-contract-definition.json @@ -0,0 +1,9 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "1", + "accessPolicyId": "aPolicy", + "contractPolicyId": "aPolicy", + "assetsSelector": [] +} diff --git a/transfer/transfer-06-file-transfer-docker/resources/create-policy.json b/transfer/transfer-06-file-transfer-docker/resources/create-policy.json new file mode 100644 index 00000000..cf7a4703 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/create-policy.json @@ -0,0 +1,14 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "aPolicy", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@type": "Set", + "permission": [], + "prohibition": [], + "obligation": [] + } +} diff --git a/transfer/transfer-06-file-transfer-docker/resources/docker-compose.yaml b/transfer/transfer-06-file-transfer-docker/resources/docker-compose.yaml new file mode 100644 index 00000000..436e2aed --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/docker-compose.yaml @@ -0,0 +1,28 @@ +version: '3.9' +services: + azurite: + container_name: azurite + image: mcr.microsoft.com/azure-storage/azurite + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + environment: + AZURITE_ACCOUNTS: provider:password + + minio: + container_name: minio + image: quay.io/minio/minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_data:/data + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: consumer + MINIO_ROOT_PASSWORD: password + +volumes: + minio_data: + driver: local \ No newline at end of file diff --git a/transfer/transfer-06-file-transfer-docker/resources/fetch-catalog.json b/transfer/transfer-06-file-transfer-docker/resources/fetch-catalog.json new file mode 100644 index 00000000..370b4f2f --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/transfer/transfer-06-file-transfer-docker/resources/negotiate-contract.json b/transfer/transfer-06-file-transfer-docker/resources/negotiate-contract.json new file mode 100644 index 00000000..82fe8036 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/negotiate-contract.json @@ -0,0 +1,15 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "ContractRequest", + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@id": "{{contract-offer-id}}", + "@type": "Offer", + "assigner": "provider", + "target": "assetId" + } +} diff --git a/transfer/transfer-06-file-transfer-docker/resources/start-transfer.json b/transfer/transfer-06-file-transfer-docker/resources/start-transfer.json new file mode 100644 index 00000000..9eaed47b --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/start-transfer.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "contractId": "{{contract-agreement-id}}", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "transferType": "AmazonS3-PUSH", + "dataDestination": { + "type": "AmazonS3", + "region": "us-west-2", + "bucketName": "src-bucket", + "objectName": "test-document.txt", + "endpointOverride": "http://localhost:9000" + } +} + diff --git a/transfer/transfer-06-file-transfer-docker/resources/test-document.txt b/transfer/transfer-06-file-transfer-docker/resources/test-document.txt new file mode 100644 index 00000000..84362ca0 --- /dev/null +++ b/transfer/transfer-06-file-transfer-docker/resources/test-document.txt @@ -0,0 +1 @@ +Test file \ No newline at end of file