From e9e41d5bdd56a2411fa2d462189faa0480ffe811 Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:26:45 +0100 Subject: [PATCH] refactor: revisit transfer samples (#140) * extracting negotiation tests * implementing Transfer02consumerPullTest * implementing Transfer03providerPushTest.java * implementing Transfer04eventConsumerTest.java * moving open telemetry to 'advanced' directory * reworking download of open telemetry jar inside gradle build file * checkStyle fixes * fix: clean up and merge * fix: added libs.edc.control.plane.api to build file * fix: fixed event consumer test * fix: resolving pr comments * refactor: move http-request-logger-wrapper.yaml to test directory * refactor: using docker image instead of docker compose for http request logger container * fix: fixing check style issues * fix: removing unnecessary stuff * refactor: removing HttpRequestLoggerUtil.java * fix: setting http-request-logger.jar archiveFilename explicitly --- README.md | 7 + advanced/README.md | 12 + .../advanced-01-open-telemetry}/README.md | 88 ++- .../contract-negotiation-trace.png | Bin .../attachments/file-transfer-trace.png | Bin .../docker-compose.yaml | 93 ++++ .../open-telemetry-consumer/Dockerfile | 11 + .../open-telemetry-consumer/build.gradle.kts | 52 +- .../open-telemetry-provider/Dockerfile | 11 + .../open-telemetry-provider/build.gradle.kts | 46 +- .../prometheus/prometheus.yml | 0 .../resources}/logging.properties | 0 .../resources/negotiate-contract.json | 14 +- .../resources/start-transfer.json | 14 + build.gradle.kts | 3 +- settings.gradle.kts | 25 +- system-tests/build.gradle.kts | 15 +- .../advanced/Advanced01openTelemetryTest.java | 86 +++ .../FileTransferCommon.java} | 11 +- .../edc/samples/common/NegotiationCommon.java | 84 +++ .../samples/common/PrerequisitesCommon.java | 104 ++++ .../transfer/Transfer00prerequisitesTest.java | 39 ++ .../transfer/Transfer01fileTransferTest.java | 80 --- .../transfer/Transfer01negotiationTest.java | 53 ++ .../transfer/Transfer02consumerPullTest.java | 95 ++++ .../Transfer02fileTransferListenerTest.java | 115 ---- .../Transfer03modifyTransferProcessTest.java | 70 --- .../transfer/Transfer03providerPushTest.java | 68 +++ .../transfer/Transfer04eventConsumerTest.java | 70 +++ .../transfer/Transfer04openTelemetryTest.java | 84 --- .../streaming/Streaming01httpToHttpTest.java | 4 +- .../streaming/Streaming02KafkaToHttpTest.java | 4 +- .../util/HttpRequestLoggerConsumer.java | 36 ++ .../util/HttpRequestLoggerContainer.java | 41 ++ .../edc/samples/util/TransferUtil.java | 115 ++++ transfer/README.md | 59 +- transfer/transfer-00-prerequisites/README.md | 96 ++++ .../connector}/build.gradle.kts | 3 +- .../resources}/certs/cert.pem | 0 .../resources}/certs/cert.pfx | Bin .../resources}/certs/key.pem | 0 .../consumer-configuration.properties | 0 .../configuration}/consumer-vault.properties | 0 .../provider-configuration.properties | 0 .../configuration}/provider-vault.properties | 0 .../register-data-plane-consumer.json | 17 + .../register-data-plane-provider.json | 17 + transfer/transfer-01-file-transfer/README.md | 300 ---------- .../file-transfer-consumer/build.gradle.kts | 46 -- .../file-transfer-consumer/config.properties | 11 - .../file-transfer-provider/build.gradle.kts | 45 -- .../file-transfer-provider/config.properties | 13 - .../filetransfer.json | 17 - .../transfer-file-local/build.gradle.kts | 32 -- .../extension/api/FileTransferDataSink.java | 82 --- .../api/FileTransferDataSinkFactory.java | 65 --- .../extension/api/FileTransferDataSource.java | 54 -- .../api/FileTransferDataSourceFactory.java | 58 -- .../extension/api/FileTransferExtension.java | 108 ---- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - transfer/transfer-01-negotiation/README.md | 242 ++++++++ .../resources/create-asset.json | 18 + .../resources/create-contract-definition.json | 9 + .../resources/create-policy.json | 13 + .../resources/fetch-catalog.json | 7 + .../resources/negotiate-contract.json} | 14 +- transfer/transfer-02-consumer-pull/README.md | 131 +++++ .../resources/start-transfer.json | 14 + .../README.md | 91 --- .../build.gradle.kts | 48 -- .../config.properties | 9 - ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - .../README.md | 87 --- .../build.gradle.kts | 46 -- .../config.properties | 6 - .../simulator/build.gradle.kts | 22 - .../transfer/TransferSimulationExtension.java | 82 --- ...rg.eclipse.edc.spi.system.ServiceExtension | 16 - .../watchdog/build.gradle.kts | 23 - .../sample/extension/watchdog/Watchdog.java | 72 --- .../extension/watchdog/WatchdogExtension.java | 48 -- ...rg.eclipse.edc.spi.system.ServiceExtension | 16 - transfer/transfer-03-provider-push/README.md | 69 +++ .../resources/start-transfer.json | 15 + transfer/transfer-04-event-consumer/README.md | 112 ++++ .../consumer-with-listener}/build.gradle.kts | 11 +- .../listener/build.gradle.kts | 0 .../TransferProcessStartedListener.java} | 26 +- ...nsferProcessStartedListenerExtension.java} | 5 +- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../docker-compose.yaml | 78 --- .../filetransfer.json | 17 - .../open-telemetry-consumer/Dockerfile | 11 - .../open-telemetry-provider/Dockerfile | 11 - .../transfer-06-consumer-pull-http/README.md | 517 ------------------ .../transfer-07-provider-push-http/README.md | 445 --------------- .../certs/cert.pem | 21 - .../certs/cert.pfx | Bin 2477 -> 0 bytes .../certs/key.pem | 28 - .../consumer-configuration.properties | 17 - .../consumer-vault.properties | 1 - .../provider-configuration.properties | 18 - .../provider-vault.properties | 1 - util/http-request-logger/Dockerfile | 12 + util/http-request-logger/build.gradle.kts | 1 + 105 files changed, 1909 insertions(+), 3097 deletions(-) create mode 100644 advanced/README.md rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/README.md (69%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/attachments/contract-negotiation-trace.png (100%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/attachments/file-transfer-trace.png (100%) create mode 100644 advanced/advanced-01-open-telemetry/docker-compose.yaml create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/open-telemetry-consumer/build.gradle.kts (63%) create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/open-telemetry-provider/build.gradle.kts (65%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/prometheus/prometheus.yml (100%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry/resources}/logging.properties (100%) rename transfer/transfer-01-file-transfer/contractoffer.json => advanced/advanced-01-open-telemetry/resources/negotiate-contract.json (57%) create mode 100644 advanced/advanced-01-open-telemetry/resources/start-transfer.json create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java rename system-tests/src/test/java/org/eclipse/edc/samples/{transfer/FileTransferSampleTestCommon.java => common/FileTransferCommon.java} (96%) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java create mode 100644 transfer/transfer-00-prerequisites/README.md rename transfer/{transfer-06-consumer-pull-http/http-pull-connector => transfer-00-prerequisites/connector}/build.gradle.kts (94%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/cert.pem (100%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/cert.pfx (100%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/key.pem (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-consumer => transfer-00-prerequisites/resources/configuration}/consumer-configuration.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-consumer => transfer-00-prerequisites/resources/configuration}/consumer-vault.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-provider => transfer-00-prerequisites/resources/configuration}/provider-configuration.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-provider => transfer-00-prerequisites/resources/configuration}/provider-vault.properties (100%) create mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json create mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json delete mode 100644 transfer/transfer-01-file-transfer/README.md delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-provider/config.properties delete mode 100644 transfer/transfer-01-file-transfer/filetransfer.json delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 transfer/transfer-01-negotiation/README.md create mode 100644 transfer/transfer-01-negotiation/resources/create-asset.json create mode 100644 transfer/transfer-01-negotiation/resources/create-contract-definition.json create mode 100644 transfer/transfer-01-negotiation/resources/create-policy.json create mode 100644 transfer/transfer-01-negotiation/resources/fetch-catalog.json rename transfer/{transfer-04-open-telemetry/contractoffer.json => transfer-01-negotiation/resources/negotiate-contract.json} (57%) create mode 100644 transfer/transfer-02-consumer-pull/README.md create mode 100644 transfer/transfer-02-consumer-pull/resources/start-transfer.json delete mode 100644 transfer/transfer-02-file-transfer-listener/README.md delete mode 100644 transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts delete mode 100644 transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties delete mode 100644 transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-03-modify-transferprocess/README.md delete mode 100644 transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 transfer/transfer-03-provider-push/README.md create mode 100644 transfer/transfer-03-provider-push/resources/start-transfer.json create mode 100644 transfer/transfer-04-event-consumer/README.md rename transfer/{transfer-07-provider-push-http/http-push-connector => transfer-04-event-consumer/consumer-with-listener}/build.gradle.kts (86%) rename transfer/{transfer-02-file-transfer-listener => transfer-04-event-consumer}/listener/build.gradle.kts (100%) rename transfer/{transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java => transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java} (52%) rename transfer/{transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java => transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java} (82%) create mode 100644 transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-04-open-telemetry/docker-compose.yaml delete mode 100644 transfer/transfer-04-open-telemetry/filetransfer.json delete mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile delete mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile delete mode 100644 transfer/transfer-06-consumer-pull-http/README.md delete mode 100644 transfer/transfer-07-provider-push-http/README.md delete mode 100644 transfer/transfer-07-provider-push-http/certs/cert.pem delete mode 100644 transfer/transfer-07-provider-push-http/certs/cert.pfx delete mode 100644 transfer/transfer-07-provider-push-http/certs/key.pem delete mode 100644 transfer/transfer-07-provider-push-http/http-push-consumer/consumer-configuration.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-consumer/consumer-vault.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-provider/provider-configuration.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-provider/provider-vault.properties create mode 100644 util/http-request-logger/Dockerfile diff --git a/README.md b/README.md index f671360a..865b99fb 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,13 @@ EDC and run different transfer scenarios. Click the link above to learn about th All transfer samples are located in the `transfer` directory. +### [Advanced](./advanced/README.md) + +Collection of advanced topics regarding the EDC framework. +Click the link above to learn about the transfer samples in more detail. + +All transfer samples are located in the `advanced` directory. + ## Contributing See [how to contribute](CONTRIBUTING.md). diff --git a/advanced/README.md b/advanced/README.md new file mode 100644 index 00000000..a9a645b3 --- /dev/null +++ b/advanced/README.md @@ -0,0 +1,12 @@ +# Advanced samples + +The samples in this scope teach advanced topics about the EDC framework. + +> Before starting with these samples, be sure to check out the [basic](../basic/README.md) and [transfer](../transfer/README.md) samples! + +## Samples + +### [Open Telemetry 01](./advanced-01-open-telemetry/README.md) Telemetry with OpenTelemetry and Micrometer + +In this sample you will learn how to generate traces with [OpenTelemetry](https://opentelemetry.io) +and collect and visualize them with [Jaeger](https://www.jaegertracing.io/). diff --git a/transfer/transfer-04-open-telemetry/README.md b/advanced/advanced-01-open-telemetry/README.md similarity index 69% rename from transfer/transfer-04-open-telemetry/README.md rename to advanced/advanced-01-open-telemetry/README.md index e35677f8..a88d08fa 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/advanced/advanced-01-open-telemetry/README.md @@ -1,11 +1,10 @@ # Telemetry with OpenTelemetry and Micrometer -This sample builds on top of [sample transfer-01-file-transfer](../transfer-01-file-transfer/README.md) to show how you -can: +This sample will show you how you can: -- generate traces with [OpenTelemetry](https://opentelemetry.io) and collect and visualize them with [Jaeger](https://www.jaegertracing.io/). +- generate traces with [OpenTelemetry](https://opentelemetry.io) and collect and visualize them with [Jaeger](https://www.jaegertracing.io/) - automatically collect metrics from infrastructure, server endpoints and client libraries with [Micrometer](https://micrometer.io) - and visualize them with [Prometheus](https://prometheus.io). + and visualize them with [Prometheus](https://prometheus.io) For this, this sample uses the Open Telemetry Java Agent, which dynamically injects bytecode to capture telemetry from several popular [libraries and frameworks](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation). @@ -27,27 +26,80 @@ is configured to expose a Prometheus metrics endpoint. To run the consumer, the provider, and Jaeger execute the following commands in the project root folder: ```bash -docker-compose -f transfer/transfer-04-open-telemetry/docker-compose.yaml up --abort-on-container-exit +docker-compose -f advanced/advanced-01-open-telemetry/docker-compose.yaml up --abort-on-container-exit ``` -Once the consumer and provider are up, start a contract negotiation by executing: +Open a new terminal. + +Register data planes for provider and consumer: + +```bash +curl -H 'Content-Type: application/json' \ + -H "X-Api-Key: password" \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ + -X POST "http://localhost:19193/management/v2/dataplanes" \ + -s | jq +``` ```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-04-open-telemetry/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" +curl -H 'Content-Type: application/json' \ + -H "X-Api-Key: password" \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json \ + -X POST "http://localhost:29193/management/v2/dataplanes" \ + -s | jq ``` -The contract negotiation causes an HTTP request sent from the consumer to the provider connector, followed by another -message from the provider to the consumer connector. Query the status of the contract negotiation by executing the -following command. Wait until the negotiation is in CONFIRMED state and note down the contract agreement id. +Create an asset: ```bash -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ + -s | jq ``` -Finally, update the contract agreement id in the `filetransfer.json` file and execute a file transfer with the following command: +Create a Policy on the provider connector: ```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-04-open-telemetry/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ + -s | jq +``` + +Follow up with the creation of a contract definition: + +```bash +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ + -s | jq +``` + +Start a contract negotiation: + +```bash +curl -H "X-Api-Key: password" \ + -H "Content-Type: application/json" \ + -d @advanced/advanced-01-open-telemetry/resources/negotiate-contract.json \ + -X POST "http://localhost:29193/management/v2/contractnegotiations" \ + -s | jq +``` + +Wait until the negotiation is in `FINALIZED` state and call + +```bash +curl -X GET -H 'X-Api-Key: password' "http://localhost:29193/management/v2/contractnegotiations/{UUID}" +``` +to get the contract agreement id. + +Finally, update the contract agreement id in the [request body](resources/start-transfer.json) and execute a file transfer with the following command: + +```bash +curl -H "X-Api-Key: password" \ + -H "Content-Type: application/json" \ + -d @advanced/advanced-01-open-telemetry/resources/start-transfer.json \ + -X POST "http://localhost:29193/management/v2/transferprocesses" ``` You can access the Jaeger UI on your browser at `http://localhost:16686`. In the search tool, we can select the service @@ -55,10 +107,10 @@ You can access the Jaeger UI on your browser at `http://localhost:16686`. In the details on the spans contained in a trace by clicking on it in the Jaeger UI. Example contract negotiation trace: -![Contract negotiation](./attachments/contract-negotiation-trace.png) +![Contract negotiation](attachments/contract-negotiation-trace.png) Example file transfer trace: -![File transfer](./attachments/file-transfer-trace.png) +![File transfer](attachments/file-transfer-trace.png) OkHttp and Jetty are part of the [libraries and frameworks](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation) that OpenTelemetry can capture telemetry from. We can observe spans related to OkHttp and Jetty as EDC uses both @@ -80,7 +132,7 @@ which has to be stored in the root folder of this sample as well. The only addit consumer: build: context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile volumes: - ./:/resources ports: @@ -128,6 +180,4 @@ it, otherwise it will use the registered global OpenTelemetry. You can look at t [ServiceLoader documentation](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/ServiceLoader.html) to have more information about service providers. ---- - -[Previous Chapter](../transfer-03-modify-transferprocess/README.md) | [Next Chapter](../transfer-05-file-transfer-cloud/README.md) +--- \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/attachments/contract-negotiation-trace.png b/advanced/advanced-01-open-telemetry/attachments/contract-negotiation-trace.png similarity index 100% rename from transfer/transfer-04-open-telemetry/attachments/contract-negotiation-trace.png rename to advanced/advanced-01-open-telemetry/attachments/contract-negotiation-trace.png diff --git a/transfer/transfer-04-open-telemetry/attachments/file-transfer-trace.png b/advanced/advanced-01-open-telemetry/attachments/file-transfer-trace.png similarity index 100% rename from transfer/transfer-04-open-telemetry/attachments/file-transfer-trace.png rename to advanced/advanced-01-open-telemetry/attachments/file-transfer-trace.png diff --git a/advanced/advanced-01-open-telemetry/docker-compose.yaml b/advanced/advanced-01-open-telemetry/docker-compose.yaml new file mode 100644 index 00000000..27d3a792 --- /dev/null +++ b/advanced/advanced-01-open-telemetry/docker-compose.yaml @@ -0,0 +1,93 @@ +version: "3.8" + +services: + + consumer: + build: + context: ../.. + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/open-telemetry + - ../../transfer/transfer-00-prerequisites/:/prerequisites + ports: + - "29193:29193" + - "29194:29194" + environment: + EDC_HOSTNAME: consumer + OTEL_SERVICE_NAME: consumer + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_METRICS_EXPORTER: prometheus + WEB_HTTP_PORT: 29191 + WEB_HTTP_PATH: /api + WEB_HTTP_PUBLIC_PORT: 29291 + WEB_HTTP_PUBLIC_PATH: /public + WEB_HTTP_CONTROL_PORT: 29192 + WEB_HTTP_CONTROL_PATH: /control + WEB_HTTP_MANAGEMENT_PORT: 29193 + WEB_HTTP_MANAGEMENT_PATH: /management + WEB_HTTP_PROTOCOL_PORT: 29194 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_CONTROL_ENDPOINT: http://consumer:29192/control + EDC_DSP_CALLBACK_ADDRESS: http://consumer:29194/protocol + EDC_PARTICIPANT_ID: consumer + EDC_API_AUTH_KEY: password + EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx + EDC_KEYSTORE_PASSWORD: 123456 + EDC_VAULT: /prerequisites/resources/configuration/provider-vault.properties + EDC_FS_CONFIG: /prerequisites/resources/configuration/provider-configuration.properties + entrypoint: java + -javaagent:/app/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/open-telemetry/resources/logging.properties + -jar /app/connector.jar + + provider: + build: + context: ../.. + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile + volumes: + - ./:/open-telemetry + - ../../transfer/transfer-00-prerequisites/:/prerequisites + ports: + - "19193:19193" + - "19192:19192" + environment: + EDC_HOSTNAME: provider + OTEL_SERVICE_NAME: provider + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + WEB_HTTP_PORT: 19191 + WEB_HTTP_PATH: /api + WEB_HTTP_PUBLIC_PORT: 19291 + WEB_HTTP_PUBLIC_PATH: /public + WEB_HTTP_CONTROL_PORT: 19192 + WEB_HTTP_CONTROL_PATH: /control + WEB_HTTP_MANAGEMENT_PORT: 19193 + WEB_HTTP_MANAGEMENT_PATH: /management + WEB_HTTP_PROTOCOL_PORT: 19194 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_CONTROL_ENDPOINT: http://provider:19192/control + EDC_DSP_CALLBACK_ADDRESS: http://provider:19194/protocol + EDC_PARTICIPANT_ID: provider + EDC_API_AUTH_KEY: password + EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx + EDC_KEYSTORE_PASSWORD: 123456 + EDC_VAULT: /prerequisites/resources/configuration/consumer-vault.properties + EDC_FS_CONFIG: /prerequisites/resources/configuration/consumer-configuration.properties + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /open-telemetry/README.md + entrypoint: java + -javaagent:/app/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/open-telemetry/resources/logging.properties + -jar /app/connector.jar + + jaeger: + image: jaegertracing/all-in-one + ports: + - "16686:16686" + + prometheus: + image: prom/prometheus:v2.30.3 + volumes: + - ./prometheus/:/etc/prometheus/ + ports: + - "9090:9090" diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile new file mode 100644 index 00000000..6641f6fa --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle advanced:advanced-01-open-telemetry:open-telemetry-consumer:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build/libs/opentelemetry-javaagent-*.jar /app/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts similarity index 63% rename from transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts rename to advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts index 03fd684a..bb8d7e95 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -26,19 +26,29 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.micrometer.core) - implementation(libs.edc.api.observability) + + implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.vault.filesystem) + implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) implementation(libs.edc.management.api) - implementation(libs.edc.dsp) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) + + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.selector.client) + + implementation(libs.edc.data.plane.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.api.observability) + implementation(libs.edc.auth.tokenbased) - runtimeOnly(libs.opentelemetry) - runtimeOnly(libs.edc.jersey.micrometer) - runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -51,25 +61,17 @@ tasks.withType { archiveFileName.set("consumer.jar") } -tasks.register("copyOpenTelemetryJar") { - doLast { - val file = file("../opentelemetry-javaagent.jar") - - if (!file.exists()) { - sourceSets["main"] - .runtimeClasspath - .files - .find { it.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get(file.path) - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } - } +tasks.register("copyOpenTelemetryJar", Copy::class) { + val openTelemetry = configurations.create("open-telemetry") + + dependencies { + openTelemetry(libs.opentelemetry) } + + from(openTelemetry) + into("build/libs") } tasks.build { finalizedBy("copyOpenTelemetryJar") -} +} \ No newline at end of file diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile b/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile new file mode 100644 index 00000000..64bbb8bb --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle advanced:advanced-01-open-telemetry:open-telemetry-provider:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-provider/build/libs/opentelemetry-javaagent-*.jar /app/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts similarity index 65% rename from transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts rename to advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts index f6f67b14..be18514c 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -27,21 +27,27 @@ dependencies { implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.api.observability) + implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) + implementation(libs.edc.vault.filesystem) - implementation(libs.edc.auth.tokenbased) + implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) - implementation(libs.edc.dsp) + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.selector.client) - implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) + implementation(libs.edc.data.plane.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.api.observability) + implementation(libs.edc.auth.tokenbased) - runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -54,23 +60,15 @@ tasks.withType { archiveFileName.set("provider.jar") } -tasks.register("copyOpenTelemetryJar") { - doLast { - val file = file("../opentelemetry-javaagent.jar") - - if (!file.exists()) { - sourceSets["main"] - .runtimeClasspath - .files - .find { it.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get(file.path) - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } - } +tasks.register("copyOpenTelemetryJar", Copy::class) { + val openTelemetry = configurations.create("open-telemetry") + + dependencies { + openTelemetry(libs.opentelemetry) } + + from(openTelemetry) + into("build/libs") } tasks.build { diff --git a/transfer/transfer-04-open-telemetry/prometheus/prometheus.yml b/advanced/advanced-01-open-telemetry/prometheus/prometheus.yml similarity index 100% rename from transfer/transfer-04-open-telemetry/prometheus/prometheus.yml rename to advanced/advanced-01-open-telemetry/prometheus/prometheus.yml diff --git a/transfer/transfer-04-open-telemetry/logging.properties b/advanced/advanced-01-open-telemetry/resources/logging.properties similarity index 100% rename from transfer/transfer-04-open-telemetry/logging.properties rename to advanced/advanced-01-open-telemetry/resources/logging.properties diff --git a/transfer/transfer-01-file-transfer/contractoffer.json b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json similarity index 57% rename from transfer/transfer-01-file-transfer/contractoffer.json rename to advanced/advanced-01-open-telemetry/resources/negotiate-contract.json index 4313071e..5497bab7 100644 --- a/transfer/transfer-01-file-transfer/contractoffer.json +++ b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json @@ -5,20 +5,20 @@ }, "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "connectorAddress": "http://provider:19194/protocol", "consumerId": "consumer", "providerId": "provider", - "connectorAddress": "http://localhost:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", + "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "assetId": "assetId", "policy": { - "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", - "@type": "set", + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "Set", "odrl:permission": [], "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "test-document" + "odrl:target": "assetId" } } -} +} \ No newline at end of file diff --git a/advanced/advanced-01-open-telemetry/resources/start-transfer.json b/advanced/advanced-01-open-telemetry/resources/start-transfer.json new file mode 100644 index 00000000..7547b17f --- /dev/null +++ b/advanced/advanced-01-open-telemetry/resources/start-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://provider:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 32d7e2fa..83279cf4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,5 +57,4 @@ allprojects { println(sourceSets["main"].runtimeClasspath.asPath) } } - -} +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 9e09f05c..48a316fa 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -34,34 +34,25 @@ include(":basic:basic-02-health-endpoint") include(":basic:basic-03-configuration") // transfer -include(":transfer:transfer-01-file-transfer:file-transfer-consumer") -include(":transfer:transfer-01-file-transfer:file-transfer-provider") -include(":transfer:transfer-01-file-transfer:transfer-file-local") +include(":transfer:transfer-00-prerequisites:connector") -include(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer") -include(":transfer:transfer-02-file-transfer-listener:listener") - -include(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer") -include(":transfer:transfer-03-modify-transferprocess:simulator") -include(":transfer:transfer-03-modify-transferprocess:watchdog") - -include(":transfer:transfer-04-open-telemetry:open-telemetry-consumer") -include(":transfer:transfer-04-open-telemetry:open-telemetry-provider") +include(":transfer:transfer-04-event-consumer:consumer-with-listener") +include(":transfer:transfer-04-event-consumer:listener") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider") include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") -include("transfer:transfer-06-consumer-pull-http:http-pull-connector") -include("transfer:transfer-07-provider-push-http:http-push-connector") - include("transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") include("transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") -include("util:http-request-logger") - +// advanced +include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") +include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") // modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") +include("util:http-request-logger") + include(":system-tests") diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index 8d033245..f7f3e6a8 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -33,12 +33,13 @@ dependencies { testCompileOnly(project(":basic:basic-01-basic-connector")) testCompileOnly(project(":basic:basic-02-health-endpoint")) testCompileOnly(project(":basic:basic-03-configuration")) - testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-consumer")) - testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider")) - testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer")) - testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer")) - testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer")) - testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider")) + + testCompileOnly(project(":transfer:transfer-00-prerequisites:connector")) + testCompileOnly(project(":transfer:transfer-04-event-consumer:consumer-with-listener")) + testCompileOnly(project(":transfer:transfer-04-event-consumer:listener")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) -} + + testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-provider")) + testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-consumer")) +} \ No newline at end of file diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java new file mode 100644 index 00000000..8a4ae1ab --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.advanced; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.createAsset; +import static org.eclipse.edc.samples.common.NegotiationCommon.createContractDefinition; +import static org.eclipse.edc.samples.common.NegotiationCommon.createPolicy; +import static org.eclipse.edc.samples.common.NegotiationCommon.getContractAgreementId; +import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +@Testcontainers +public class Advanced01openTelemetryTest { + + private static final String DOCKER_COMPOSE_YAML = "advanced/advanced-01-open-telemetry/docker-compose.yaml"; + private static final String NEGOTIATE_CONTRACT_FILE_PATH = "advanced/advanced-01-open-telemetry/resources/negotiate-contract.json"; + private static final String START_TRANSFER_FILE_PATH = "advanced/advanced-01-open-telemetry/resources/start-transfer.json"; + private static final String JAEGER_URL = "http://localhost:16686"; + + @Container + public static DockerComposeContainer environment = + new DockerComposeContainer<>(getFileFromRelativePath(DOCKER_COMPOSE_YAML)) + .withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + + @BeforeAll + static void setUp() { + environment.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + createAsset(); + createPolicy(); + createContractDefinition(); + var contractNegotiationId = negotiateContract(NEGOTIATE_CONTRACT_FILE_PATH); + var contractAgreementId = getContractAgreementId(contractNegotiationId); + var transferProcessId = startTransfer(getFileContentFromRelativePath(START_TRANSFER_FILE_PATH), contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + assertJaegerState(); + } + + private void assertJaegerState() { + try { + var url = new URL(JAEGER_URL); + var huc = (HttpURLConnection) url.openConnection(); + assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); + } catch (IOException e) { + fail("Unable to assert Jaeger state", e); + } + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java similarity index 96% rename from system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java rename to system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java index 95e64ec6..0e79a98b 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java @@ -9,10 +9,11 @@ * * Contributors: * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases * */ -package org.eclipse.edc.samples.transfer; +package org.eclipse.edc.samples.common; import com.fasterxml.jackson.databind.ObjectMapper; import io.restassured.http.ContentType; @@ -39,7 +40,7 @@ /** * Encapsulates common settings, test steps, and helper methods for transfer samples */ -public class FileTransferSampleTestCommon { +public class FileTransferCommon { static final ObjectMapper MAPPER = new ObjectMapper(); @@ -64,7 +65,7 @@ public class FileTransferSampleTestCommon { * @param sampleAssetFilePath Relative path starting from the root of the project to a file which will be read from for transfer. * @param destinationFilePath Relative path starting from the root of the project where the transferred file will be written to. */ - public FileTransferSampleTestCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { + public FileTransferCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { this.sampleAssetFilePath = sampleAssetFilePath; sampleAssetFile = getFileFromRelativePath(sampleAssetFilePath); @@ -110,7 +111,7 @@ void cleanTemporaryTestFiles() { /** * Assert that the file to be copied exists at the expected location. - * This method waits a duration which is defined in {@link FileTransferSampleTestCommon#timeout}. + * This method waits a duration which is defined in {@link FileTransferCommon#timeout}. */ void assertDestinationFileContent() { await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() @@ -184,7 +185,7 @@ void lookUpContractAgreementId() { * Assert that a POST request to initiate transfer process is successful. * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses"} * - * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}. + * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferCommon#TRANSFER_FILE_PATH}. */ String requestTransferFile(String transferFilePath) throws IOException { var transferJsonFile = getFileFromRelativePath(transferFilePath); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java new file mode 100644 index 00000000..d8780312 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.common; + + +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.util.TransferUtil.POLL_INTERVAL; +import static org.eclipse.edc.samples.util.TransferUtil.TIMEOUT; +import static org.eclipse.edc.samples.util.TransferUtil.get; +import static org.eclipse.edc.samples.util.TransferUtil.post; + +public class NegotiationCommon { + + private static final String CREATE_ASSET_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-asset.json"; + private static final String V2_ASSETS_PATH = "/v2/assets"; + private static final String CREATE_POLICY_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-policy.json"; + private static final String V2_POLICY_DEFINITIONS_PATH = "/v2/policydefinitions"; + private static final String CREATE_CONTRACT_DEFINITION_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-contract-definition.json"; + private static final String V2_CONTRACT_DEFINITIONS_PATH = "/v2/contractdefinitions"; + private static final String FETCH_CATALOG_FILE_PATH = "transfer/transfer-01-negotiation/resources/fetch-catalog.json"; + private static final String V2_CATALOG_REQUEST_PATH = "/v2/catalog/request"; + private static final String NEGOTIATE_CONTRACT_FILE_PATH = "transfer/transfer-01-negotiation/resources/negotiate-contract.json"; + private static final String V2_CONTRACT_NEGOTIATIONS_PATH = "/v2/contractnegotiations/"; + private static final String CONTRACT_NEGOTIATION_ID = "@id"; + private static final String CONTRACT_AGREEMENT_ID = "'edc:contractAgreementId'"; + + public static void createAsset() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_ASSETS_PATH, getFileContentFromRelativePath(CREATE_ASSET_FILE_PATH)); + } + + public static void createPolicy() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_POLICY_DEFINITIONS_PATH, getFileContentFromRelativePath(CREATE_POLICY_FILE_PATH)); + } + + public static void createContractDefinition() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_CONTRACT_DEFINITIONS_PATH, getFileContentFromRelativePath(CREATE_CONTRACT_DEFINITION_FILE_PATH)); + } + + public static void fetchCatalog() { + post(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CATALOG_REQUEST_PATH, getFileContentFromRelativePath(FETCH_CATALOG_FILE_PATH)); + } + + public static String negotiateContract(String negotiateContractFilePath) { + var contractNegotiationId = post(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CONTRACT_NEGOTIATIONS_PATH, getFileContentFromRelativePath(negotiateContractFilePath), CONTRACT_NEGOTIATION_ID); + assertThat(contractNegotiationId).isNotEmpty(); + return contractNegotiationId; + } + + public static String negotiateContract() { + return negotiateContract(NEGOTIATE_CONTRACT_FILE_PATH); + } + + public static String getContractAgreementId(String contractNegotiationId) { + return await() + .atMost(TIMEOUT) + .pollInterval(POLL_INTERVAL) + .until(() -> get(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CONTRACT_NEGOTIATIONS_PATH + contractNegotiationId, CONTRACT_AGREEMENT_ID), Objects::nonNull); + } + + public static String runNegotiation() { + createAsset(); + createPolicy(); + createContractDefinition(); + fetchCatalog(); + var contractNegotiationId = negotiateContract(); + return getContractAgreementId(contractNegotiationId); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java new file mode 100644 index 00000000..a799da83 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.common; + +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; + +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; + +public class PrerequisitesCommon { + public static final String API_KEY_HEADER_KEY = "X-Api-Key"; + public static final String API_KEY_HEADER_VALUE = "password"; + public static final String PROVIDER_MANAGEMENT_URL = "http://localhost:19193/management"; + public static final String CONSUMER_MANAGEMENT_URL = "http://localhost:29193/management"; + public static final String CONSUMER_PUBLIC_URL = "http://localhost:29291/public"; + + private static final String CONNECTOR_MODULE_PATH = ":transfer:transfer-00-prerequisites:connector"; + private static final String PROVIDER = "provider"; + private static final String CONSUMER = "consumer"; + private static final String EDC_KEYSTORE = "edc.keystore"; + private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password"; + private static final String EDC_VAULT = "edc.vault"; + private static final String EDC_FS_CONFIG = "edc.fs.config"; + + private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx"; + private static final String KEYSTORE_PASSWORD = "123456"; + private static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties"; + private static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties"; + private static final String PROVIDER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties"; + private static final String CONSUMER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties"; + private static final String REGISTER_DATA_PLANE_PROVIDER_JSON = "transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json"; + private static final String REGISTER_DATA_PLANE_CONSUMER_JSON = "transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json"; + private static final String V2_DATAPLANES_PATH = "/v2/dataplanes"; + + public static EdcRuntimeExtension getProvider() { + return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_VAULT_PROPERTIES_FILE_PATH, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); + + } + + public static EdcRuntimeExtension getConsumer() { + return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + } + + public static EdcRuntimeExtension getConsumer(String modulePath) { + return getConnector(modulePath, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + } + + public static void registerDataPlaneProvider() { + registerDataPlane(PROVIDER_MANAGEMENT_URL, REGISTER_DATA_PLANE_PROVIDER_JSON); + } + + public static void runPrerequisites() { + registerDataPlaneProvider(); + } + + private static EdcRuntimeExtension getConnector( + String modulePath, + String moduleName, + String vaultPropertiesFilePath, + String configPropertiesFilePath + ) { + return new EdcRuntimeExtension( + modulePath, + moduleName, + Map.of( + EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(), + EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD, + EDC_VAULT, getFileFromRelativePath(vaultPropertiesFilePath).getAbsolutePath(), + EDC_FS_CONFIG, getFileFromRelativePath(configPropertiesFilePath).getAbsolutePath() + ) + ); + } + + private static void registerDataPlane(String host, String payloadPath) { + var requestBody = getFileFromRelativePath(payloadPath); + + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(host + V2_DATAPLANES_PATH) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java new file mode 100644 index 00000000..b05bb404 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.registerDataPlaneProvider; + +@EndToEndTest +public class Transfer00prerequisitesTest { + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Test + void runSampleSteps() { + registerDataPlaneProvider(); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java deleted file mode 100644 index f3cf5a53..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Map; - -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; - -@EndToEndTest -public class Transfer01fileTransferTest { - - private static final String SAMPLE_FOLDER = "transfer/transfer-01-file-transfer"; - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = SAMPLE_FOLDER + "/file-transfer-consumer/config.properties"; - static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = SAMPLE_FOLDER + "/file-transfer-provider/config.properties"; - static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/requested.test.txt"; - - @RegisterExtension - static EdcRuntimeExtension provider = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-provider", - "provider", - Map.of( - // Override 'edc.samples.transfer.01.asset.path' implicitly set via property 'edc.fs.config'. - "edc.samples.transfer.01.asset.path", getFileFromRelativePath(SAMPLE_ASSET_FILE_PATH).getAbsolutePath(), - "edc.fs.config", getFileFromRelativePath(PROVIDER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() throws Exception { - testUtils.assertTestPrerequisites(); - - testUtils.initiateContractNegotiation(); - testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(); - testUtils.assertDestinationFileContent(); - - testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); - } - - @AfterEach - protected void tearDown() { - testUtils.cleanTemporaryTestFiles(); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java new file mode 100644 index 00000000..b9b10f15 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.NegotiationCommon.createAsset; +import static org.eclipse.edc.samples.common.NegotiationCommon.createContractDefinition; +import static org.eclipse.edc.samples.common.NegotiationCommon.createPolicy; +import static org.eclipse.edc.samples.common.NegotiationCommon.fetchCatalog; +import static org.eclipse.edc.samples.common.NegotiationCommon.getContractAgreementId; +import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; + +@EndToEndTest +public class Transfer01negotiationTest { + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Test + void runSampleSteps() { + runPrerequisites(); + createAsset(); + createPolicy(); + createContractDefinition(); + fetchCatalog(); + var contractNegotiationId = negotiateContract(); + var contractAgreementId = getContractAgreementId(contractNegotiationId); + assertThat(contractAgreementId).isNotEmpty(); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java new file mode 100644 index 00000000..49523807 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerConsumer; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static io.restassured.RestAssured.given; +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_PUBLIC_URL; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.not; + +@EndToEndTest +@Testcontainers +public class Transfer02consumerPullTest { + + private static final HttpRequestLoggerConsumer LOG_CONSUMER = new HttpRequestLoggerConsumer(); + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-02-consumer-pull/resources/start-transfer.json"; + private static final String AUTH_CODE_KEY = "authCode"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Container + public static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(LOG_CONSUMER); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var contractAgreementId = runNegotiation(); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + var authCode = LOG_CONSUMER.getJsonValue(AUTH_CODE_KEY); + checkData(authCode); + } + + private static void checkData(String authCode) { + var result = given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE, AUTHORIZATION, authCode) + .when() + .get(CONSUMER_PUBLIC_URL) + .then() + .statusCode(HttpStatus.SC_OK) + .log() + .ifError() + .body("[0].name", not(emptyString())) + .extract() + .jsonPath() + .get("[0].name"); + + assertThat(result).isEqualTo("Leanne Graham"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java deleted file mode 100644 index 8de5103b..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.io.File; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; - -@EndToEndTest -public class Transfer02fileTransferListenerTest { - - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties"; - static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-01-file-transfer/file-transfer-provider/config.properties"; - // Reuse an already existing file for the test. Could be set to any other existing file in the repository. - static final String SAMPLE_ASSET_FILE_PATH = "transfer/transfer-02-file-transfer-listener/README.md"; - static final String DESTINATION_FILE_PATH = "transfer/transfer-02-file-transfer-listener/requested.test.txt"; - // marker.txt is a fixed name and always in the same directory as the file defined with DESTINATION_FILE_PATH. - static final String MARKER_FILE_PATH = "transfer/transfer-02-file-transfer-listener/marker.txt"; - static final File MARKER_FILE = getFileFromRelativePath(MARKER_FILE_PATH); - static final String MARKER_FILE_CONTENT = "Transfer complete"; - @RegisterExtension - static EdcRuntimeExtension provider = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-provider", - "provider", - Map.of( - // Override 'edc.samples.transfer.01.asset.path' implicitly set via property 'edc.fs.config'. - "edc.samples.transfer.01.asset.path", getFileFromRelativePath(SAMPLE_ASSET_FILE_PATH).getAbsolutePath(), - "edc.fs.config", getFileFromRelativePath(PROVIDER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - private final FileTransferSampleTestCommon common = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - @AfterEach - void tearDown() { - cleanTemporaryTestFiles(); - } - - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() throws Exception { - assertTestPrerequisites(); - - common.initiateContractNegotiation(); - common.lookUpContractAgreementId(); - common.requestTransferFile(); - common.assertDestinationFileContent(); - assertFileContent(MARKER_FILE, MARKER_FILE_CONTENT); - } - - /** - * Assert that prerequisites are fulfilled before running the test. - * This assertion checks only whether the file to be copied is not existing already. - */ - void assertTestPrerequisites() { - common.assertTestPrerequisites(); - - assertThat(MARKER_FILE).doesNotExist(); - } - - /** - * Remove files created while running the tests. - * The copied file and the marker file will be deleted. - */ - void cleanTemporaryTestFiles() { - common.cleanTemporaryTestFiles(); - - MARKER_FILE.delete(); - } - - /** - * Assert that the marker file has been created at the expected location with the expected content. - * This method waits a duration which is defined in {@link FileTransferSampleTestCommon#timeout}. - */ - void assertFileContent(File markerFile, @NotNull String markerFileContent) { - await().atMost(common.timeout).pollInterval(common.pollInterval).untilAsserted(() - -> assertThat(markerFile).hasContent(markerFileContent)); - } - - -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java deleted file mode 100644 index ebd2eab2..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.time.Duration; -import java.util.Map; - -import static io.restassured.RestAssured.given; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.API_KEY_HEADER_KEY; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.API_KEY_HEADER_VALUE; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.MANAGEMENT_API_URL; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; -import static org.hamcrest.CoreMatchers.is; - -@EndToEndTest -public class Transfer03modifyTransferProcessTest { - - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties"; - static final Duration DURATION = Duration.ofSeconds(15); - static final Duration POLL_INTERVAL = Duration.ofMillis(500); - - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - /** - * Requests transfer processes from management API and check for expected changes on the transfer process. - */ - @Test - void runSample() { - await().atMost(DURATION).pollInterval(POLL_INTERVAL).untilAsserted(() -> { - given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .contentType(ContentType.JSON) - .post(MANAGEMENT_API_URL + "/v2/transferprocesses/request") - .then() - .statusCode(HttpStatus.SC_OK) - .contentType(ContentType.JSON) - .body("[0].@id", is("tp-sample-transfer-03")) - .body("[0].'edc:state'", is("TERMINATING")) - .body("[0].'edc:errorDetail'", is("timeout by watchdog")); - }); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java new file mode 100644 index 00000000..a607c159 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerConsumer; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +@Testcontainers +public class Transfer03providerPushTest { + + private static final HttpRequestLoggerConsumer LOG_CONSUMER = new HttpRequestLoggerConsumer(); + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-03-provider-push/resources/start-transfer.json"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Container + public static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(LOG_CONSUMER); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + var contractAgreementId = runNegotiation(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.COMPLETED); + assertThat(LOG_CONSUMER.toUtf8String()).contains("Leanne Graham"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java new file mode 100644 index 00000000..60f42781 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +public class Transfer04eventConsumerTest { + private static final String CONSUMER_WITH_LISTENER_MODULE_PATH = ":transfer:transfer-04-event-consumer:consumer-with-listener"; + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-02-consumer-pull/resources/start-transfer.json"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(CONSUMER_WITH_LISTENER_MODULE_PATH); + + @Container + static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + var standardOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(standardOutputStream)); + runPrerequisites(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var contractAgreementId = runNegotiation(); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + var standardOutput = standardOutputStream.toString(); + assertThat(standardOutput).contains("TransferProcessStartedListener received STARTED event"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java deleted file mode 100644 index e562cb12..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Sample workflow test - * - */ - -package org.eclipse.edc.samples.transfer; - -import org.apache.http.HttpStatus; -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; - -@EndToEndTest -@Testcontainers -public class Transfer04openTelemetryTest { - - private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; - private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; - private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; - private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; - private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; - private static final String JAEGER_URL = "http://localhost:16686"; - - private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - @Container - public static DockerComposeContainer environment = - new DockerComposeContainer<>(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) - .withLocalCompose(true) - .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); - - @BeforeAll - static void setUp() { - environment.start(); - } - - @Test - void runSampleSteps() throws Exception { - testUtils.assertTestPrerequisites(); - testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); - testUtils.assertDestinationFileContent(); - testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); - assertJaegerState(); - } - - private void assertJaegerState() { - try { - var url = new URL(JAEGER_URL); - var huc = (HttpURLConnection) url.openConnection(); - assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); - } catch (IOException e) { - fail("Unable to assert Jaeger state", e); - } - } - - @AfterEach - protected void tearDown() { - testUtils.cleanTemporaryTestFiles(); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java index 0259d0f1..12b70552 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java @@ -37,8 +37,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileContentFromRelativePath; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; @EndToEndTest public class Streaming01httpToHttpTest { diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java index 21708542..de876440 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java @@ -44,8 +44,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileContentFromRelativePath; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; @Testcontainers @EndToEndTest diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java new file mode 100644 index 00000000..687612d7 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import org.testcontainers.containers.output.ToStringConsumer; + +import java.util.regex.Pattern; + +import static java.lang.String.format; +import static org.apache.commons.lang3.StringUtils.EMPTY; + +public class HttpRequestLoggerConsumer extends ToStringConsumer { + + private static final String REGEX_FORMAT = "(?<=\"%s\":\")[^\"]*"; + + public String getJsonValue(String key) { + var pattern = Pattern.compile(format(REGEX_FORMAT, key)); + var matcher = pattern.matcher(toUtf8String()); + if (matcher.find()) { + return matcher.group(); + } + return EMPTY; + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java new file mode 100644 index 00000000..9f7a7792 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.ToStringConsumer; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.util.List; + +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; + +public class HttpRequestLoggerContainer extends GenericContainer { + + private static final String HTTP_REQUEST_LOGGER_DOCKERFILE_PATH = "util/http-request-logger/Dockerfile"; + private static final ImageFromDockerfile IMAGE_FROM_DOCKERFILE = new ImageFromDockerfile() + .withDockerfile(getFileFromRelativePath(HTTP_REQUEST_LOGGER_DOCKERFILE_PATH).toPath()); + private static final String PORT_BINDING = "4000:4000"; + + public HttpRequestLoggerContainer() { + super(IMAGE_FROM_DOCKERFILE); + this.setPortBindings(List.of(PORT_BINDING)); + } + + public HttpRequestLoggerContainer(ToStringConsumer toStringConsumer) { + this(); + this.setLogConsumers(List.of(toStringConsumer)); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java new file mode 100644 index 00000000..34189abe --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; + +import java.time.Duration; + +import static io.restassured.RestAssured.given; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_MANAGEMENT_URL; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.not; + +public class TransferUtil { + + public static final Duration TIMEOUT = Duration.ofSeconds(30); + public static final Duration POLL_DELAY = Duration.ofMillis(1000); + public static final Duration POLL_INTERVAL = Duration.ofMillis(500); + + private static final String TRANSFER_PROCESS_ID = "@id"; + private static final String CONTRACT_AGREEMENT_ID_KEY = ""; + private static final String V2_TRANSFER_PROCESSES_PATH = "/v2/transferprocesses/"; + private static final String EDC_STATE = "'edc:state'"; + + public static void get(String url) { + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .when() + .get(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } + + public static String get(String url, String jsonPath) { + return given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .when() + .get(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK) + .body(jsonPath, not(emptyString())) + .extract() + .jsonPath() + .get(jsonPath); + } + + public static void post(String url, String requestBody) { + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } + + public static String post(String url, String requestBody, String jsonPath) { + return given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK) + .body(jsonPath, not(emptyString())) + .extract() + .jsonPath() + .get(jsonPath); + } + + public static String startTransfer(String requestBody, String contractAgreementId) { + requestBody = requestBody.replaceAll(CONTRACT_AGREEMENT_ID_KEY, contractAgreementId); + return post(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH, requestBody, TRANSFER_PROCESS_ID); + } + + public static void checkTransferStatus(String transferProcessId, TransferProcessStates status) { + await() + .atMost(TIMEOUT) + .pollDelay(POLL_DELAY) + .pollInterval(POLL_INTERVAL) + .until( + () -> get(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH + transferProcessId, EDC_STATE), + (result) -> status.name().equals(result) + ); + } +} diff --git a/transfer/README.md b/transfer/README.md index a6e2b524..de66133b 100644 --- a/transfer/README.md +++ b/transfer/README.md @@ -2,38 +2,39 @@ The samples in this scope revolve around the topic of transferring data between two connectors. Here you will learn about the steps required for a transfer on provider as well as consumer side. The -samples start with the simple example of a local file transfer and then show different ways to tweak -that transfer, before a transfer is performed between different cloud providers. +samples start with the simple example of a building and configuring the connector library. +Furthermore, you will learn about the contract negotiation workflow and how to perform a data transfer. > Before starting with these samples, be sure to check out the [basic samples](../basic/README.md)! ## Samples -### [Transfer sample 01](./transfer-01-file-transfer/README.md): Perform a local file transfer +### [Transfer sample 00](./transfer-00-prerequisites/README.md): Prerequisites -In this sample you will perform your first data transfer. To keep it simple, a file is transferred -on your local machine from one directory to another. You will see which extensions and -configurations are required for a transfer and learn -how to create a data offer as a provider as well as which steps to perform as a consumer. +This is a prerequisite for the following chapters. +You will build the connector library here, configure and execute it. -### [Transfer sample 02](./transfer-02-file-transfer-listener/README.md): Implement a transfer listener +### [Transfer sample 01](./transfer-01-negotiation/README.md): Negotiation -As you'll learn in the first transfer sample, the process of a data transfer is executed in a state -machine and runs asynchronously in the background after being initiated. This sample is an -enhancement of the previous sample and shows how a listener can be used to immediately react to -state changes in this asynchronous process. +Before two connectors can exchange actual data, negotiation has to take place. +The final goal of this example is to showcase the negotiation workflow between two connectors so that +the actual data transfer can take place. This chapter is a prerequisite to the following chapters. -### [Transfer sample 03](./transfer-03-modify-transferprocess/README.md): Modify a TransferProcess +### [Transfer sample 02](./transfer-02-consumer-pull/README.md): Perform a consumer pull exchange between a consumer and a provider -This sample is another enhancement of the first transfer sample. After you've learned how to react -to state changes during a data transfer, here you will see how to actively modify a process in the -state machine in a thread-safe manner. +In this sample you will perform your first actual data transfer. +The purpose of this sample is to show a data exchange between 2 connectors, one representing the +data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find +more details on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). -### [Transfer sample 04](./transfer-04-open-telemetry/README.md): Open Telemetry +### [Transfer sample 03](transfer-03-provider-push/README.md): Perform a provider push exchange between a consumer and a provider -Now that you've gotten familiar with the process of transferring data, this sample will show -how `OpenTelemetry`,`Jaeger`, `Prometheus` and `Micrometer` can be used to collect and visualize -traces and metrics during this process. +This sample demonstrates the "provider push" use case that you can find more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +### [Transfer sample 04](transfer-04-event-consumer/README.md): Consuming connector events + +In this sample you will learn how to react to certain connector events. ### [Transfer sample 05](./transfer-05-file-transfer-cloud/README.md): Perform a file transfer between cloud providers @@ -43,20 +44,4 @@ transfer scenario, where a file is transferred not in the local file system, but different cloud providers. In this sample you will set up a provider that offers a file located in an `Azure Blob Storage`, and a consumer that requests to transfer this file to an `AWS S3 bucket`. Terraform is used for creating all required cloud -resources. - -### [Transfer sample 06](./transfer-06-consumer-pull-http/README.md): Perform a consumer pull exchange between a consumer and a provider - -In this sample, we will describe a step-by-step guide to demonstrate a consumer pull exchange -between two connections. One connecter is a consumer and the other is a provider. The consumer will -initiate a transfer, and the provider will send an EndpointDataReference to the consumer. Finally, -the consumer will be able to access the data by requesting the endpoint that received through the -EndpointDataReference. - -### [Transfer sample 07](./transfer-07-provider-push-http/README.md): Perform a provider push exchange between a consumer and a provider - -In this sample, we will describe a step-by-step guide to demonstrate a provider push exchange -between two connections. One connecter is a consumer and the other is a provider. The consumer will -initiate the transfer by sending a DataRequest with any destination type other -than HttpProxy, and the provider will fetch the date from the actual DataSource and push it to the -consumer. +resources. \ No newline at end of file diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md new file mode 100644 index 00000000..4e8a968d --- /dev/null +++ b/transfer/transfer-00-prerequisites/README.md @@ -0,0 +1,96 @@ +# Prerequisites + +The purpose of this example is to make preparations for a connector-to-connector communication. +For that we'll set up two connectors. + +One connector will be the "consumer" connector while the other will act as the +"provider" connector. More on that in the following chapters. + +For the sake of simplicity, the provider and the consumer +will run on the same machine, but in a real world configuration, they will likely be on different +machines. + +This sample will go through: + +* Building the connector module +* Running the provider connector +* Running the consumer connector +* Registering data plane instance for the provider connector + +## Run the sample + +### 1. Build the connector + +When we speak of a connector we actually mean a .jar file that is launched on a machine. +Before we can launch a connector we'll have to build the .jar file. + +Execute this command in project root: + +```bash +./gradlew transfer:transfer-00-prerequisites:connector:build +``` + +After the build end you should verify that the connector.jar is created in the directory +[/connector/build/libs/connector.jar](connector/build/libs/connector.jar) + +We can use the same .jar file for both connectors. Note that the consumer and provider connectors differ in their configuration. + +Inspect the different configuration files below: + +* [provider-configuration.properties](resources/configuration/provider-configuration.properties) +* [consumer-configuration.properties](resources/configuration/consumer-configuration.properties) + +The section bellow will show you some explanation about some of the properties that you can find in +the configuration files. + +#### 1. edc.receiver.http.endpoint + +This property is used to define the endpoint where the connector consumer will send the +EndpointDataReference. + +#### 2. edc.dataplane.token.validation.endpoint + +This property is used to define the endpoint exposed by the control plane to validate the token. + +### 2. Run the connectors + +To run the provider, just run the following command + +```bash +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +``` + +To run the consumer, just run the following command (different terminal) + +```bash +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +``` + +Assuming you didn't change the ports in config files, the consumer will listen on the +ports `29191`, `29192` (management API) and `29292` (DSP API) and the provider will listen on the +ports `12181`, `19182` (management API) and `19282` (DSP API). + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 3. Register data plane instance for provider + +Before a consumer can start talking to a provider, it is necessary to register the data plane +instance of a connector. This is done by sending a POST request to the management API of the +provider connector. The [request body](resources/dataplane/register-data-plane-provider.json) should contain the data plane instance of the consumer +connector. + +The registration of the provider data plane instance is done by sending a POST +request to the management API of the connector. + +Open a new terminal and execute: + +```bash +curl -H 'Content-Type: application/json' \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ + -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq +``` + +The connectors have been configured successfully and are ready to be used. + +[Next Chapter](../transfer-01-negotiation/README.md) \ No newline at end of file diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts b/transfer/transfer-00-prerequisites/connector/build.gradle.kts similarity index 94% rename from transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts rename to transfer/transfer-00-prerequisites/connector/build.gradle.kts index 9986a7a4..216f0f6e 100644 --- a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts +++ b/transfer/transfer-00-prerequisites/connector/build.gradle.kts @@ -20,6 +20,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) @@ -47,6 +48,6 @@ var distZip = tasks.getByName("distZip") tasks.withType { mergeServiceFiles() - archiveFileName.set("pull-connector.jar") + archiveFileName.set("connector.jar") dependsOn(distTar, distZip) } diff --git a/transfer/transfer-06-consumer-pull-http/certs/cert.pem b/transfer/transfer-00-prerequisites/resources/certs/cert.pem similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/cert.pem rename to transfer/transfer-00-prerequisites/resources/certs/cert.pem diff --git a/transfer/transfer-06-consumer-pull-http/certs/cert.pfx b/transfer/transfer-00-prerequisites/resources/certs/cert.pfx similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/cert.pfx rename to transfer/transfer-00-prerequisites/resources/certs/cert.pfx diff --git a/transfer/transfer-06-consumer-pull-http/certs/key.pem b/transfer/transfer-00-prerequisites/resources/certs/key.pem similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/key.pem rename to transfer/transfer-00-prerequisites/resources/certs/key.pem diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties rename to transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties rename to transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties rename to transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties rename to transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json new file mode 100644 index 00000000..73045bc6 --- /dev/null +++ b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json @@ -0,0 +1,17 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "http-pull-consumer-dataplane", + "url": "http://localhost:29192/control/transfer", + "allowedSourceTypes": [ + "HttpData" + ], + "allowedDestTypes": [ + "HttpProxy", + "HttpData" + ], + "properties": { + "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" + } +} \ No newline at end of file diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json new file mode 100644 index 00000000..6b90e706 --- /dev/null +++ b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json @@ -0,0 +1,17 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "http-pull-provider-dataplane", + "url": "http://localhost:19192/control/transfer", + "allowedSourceTypes": [ + "HttpData" + ], + "allowedDestTypes": [ + "HttpProxy", + "HttpData" + ], + "properties": { + "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" + } +} \ No newline at end of file diff --git a/transfer/transfer-01-file-transfer/README.md b/transfer/transfer-01-file-transfer/README.md deleted file mode 100644 index 043fe1b3..00000000 --- a/transfer/transfer-01-file-transfer/README.md +++ /dev/null @@ -1,300 +0,0 @@ -# Implement a simple file transfer - -After successfully providing custom configuration properties to the EDC, we will perform a data transfer next: transmit -a test file from one connector to another connector. We want to keep things simple, so we will run both connectors on -the same physical machine (i.e. your development machine) and the file is transferred from one folder in the file system -to another folder. It is not difficult to imagine that instead of the local file system, the transfer happens between -more sophisticated storage locations, like a database or a cloud storage. - -This is quite a big step up from the previous sample, where we ran only one connector. Those are the concrete tasks: - -* Creating an additional connector, so that in the end we have two connectors, a consumer and a provider -* Providing communication between provider and consumer using DSP messages -* Utilizing the management API to interact with the connector system -* Performing a contract negotiation between provider and consumer -* Performing a file transfer - * The consumer will initiate a file transfer - * The provider will fulfill that request and copy a file to the desired location - -Also, in order to keep things organized, the code in this example has been separated into several Java modules: - -* `file-transfer-[consumer|provider]`: contains the configuration and build files for both the consumer and the provider connector -* `transfer-file-local`: contains all the code necessary for the file transfer, integrated on provider side -* `status-checker`: contains the code for checking if the file has been transfer, integrated on the consumer side - -## Create the file transfer extension - -The provider connector needs to transfer a file to the location specified by the consumer connector when the data is -requested. In order to offer any data, the provider must maintain an internal list of assets that are available for -transfer, the so-called "catalog". For the sake of simplicity we use an in-memory catalog and pre-fill it with just one -single class. The provider also needs to create a contract offer for the asset, based on which a contract agreement can -be negotiated. For this, we also use an in-memory store and add a single contract definition that is valid for the -asset. - -```java -// in FileTransferExtension.java -@Override -public void initialize(ServiceExtensionContext context){ - // ... - var policy = createPolicy(); - policyStore.save(policy); - - registerDataEntries(context); - registerContractDefinition(policy.getUid()); - // ... -} - -//... - -private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); - var assetPath = Path.of(assetPathSetting); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("path", assetPath.getParent().toString()) - .property("filename", assetPath.getFileName().toString()) - .build(); - - var assetId = "test-document"; - var asset = Asset.Builder.newInstance().id(assetId).build(); - - assetIndex.create(asset, dataAddress); - } - -private void registerContractDefinition(String uid) { - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(uid) - .contractPolicyId(uid) - .assetsSelectorCriterion(criterion(Asset.PROPERTY_ID, "=", "1")) - .whenEquals(Asset.PROPERTY_ID, "test-document") - .build()) - .build(); - - contractStore.save(contractDefinition); - } -``` - -This adds an `Asset` to the `AssetIndex` and the relative `DataAddress` to the `DataAddressResolver`. -Or, in other words, your provider now "hosts" one file named `test-document.txt` located in the path -configured by the setting `edc.samples.transfer.01.asset.path` on your development machine. It makes it available for -transfer under its `id` `"test-document"`. While it makes sense to have some sort of similarity between file name and -id, it is by no means mandatory. - -It also adds a `ContractDefinition` with `id` `1` and a previously created `Policy` (code omitted above), that poses no -restrictions on the data usage. The `ContractDefinition` also has an `assetsSelector` `Criterion` defining that it is -valid for all assets with the `id` `test-document`. Thus, it is valid for the created asset. - -Next to offering the file, the provider also needs to be able to transfer the file. Therefore, the `transfer-file` -module also provides the code for copying the file to a specified location (code omitted here for brevity). It contains -the [FileTransferDataSource](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java) -and the [FileTransferDataSink](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java) -as well as respective factories for both. The factories are registered with the `PipelineService` in the -[FileTransferExtension](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java), -thus making them available when a data request is processed. - -## Create the status checker extension - -The consumer needs to know when the file transfer has been completed. For doing that, in the extension -we are going to implement a custom `StatusChecker` that will be registered with the `StatusCheckerRegistry` in the -[SampleStatusCheckerExtension](status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java) -The custom status checker will handle the check for the destination type `File` and it will check that the path -specified in the data requests exists. The code is available in the -class [SampleFileStatusChecker](status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java) - -## Create the connectors - -After creating the required extensions, we next need to create the two connectors. For both of them we need a gradle -build file and a config file. Common dependencies we need to add to the build files on both sides are the following: - -```kotlin -// in file-transfer-consumer/build.gradle.kts and file-transfer-provider/build.gradle.kts: -implementation(libs.edc.configuration.filesystem) - -implementation(libs.edc.dsp) -implementation(libs.edc.iam.mock) - -implementation(libs.edc.management.api) -implementation(libs.edc.auth.tokenbased) -``` - -Three of these dependencies are new and have not been used in the previous samples: -1. `data-protocols:dsp`: contains all DSP modules and therefore enables DSP communication with other connectors -2. `extensions:iam:iam-mock`: provides a no-op identity provider, which does not require certificates and performs no checks -3. `extensions:api:auth-tokenbased`: adds authentication for management API endpoints - -### Provider connector - -As the provider connector is the one performing the file transfer after the file has been requested by the consumer, it -needs the `transfer-file-local` extension provided in this sample. - -```kotlin -implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) -``` - -We also need to adjust the provider's `config.properties`. The property `edc.samples.transfer.01.asset.path` should -point to an existing file in our local environment, as this is the file that will be transferred. We also configure a -separate API context for the management API, like we learned in previous chapter. Then we add the property -`edc.dsp.callback.address`, which should point to our provider connector's DSP address. This is used as the callback -address during the contract negotiation. Since the DSP API is running on a different port (default is `8282`), we set -the webhook address to `http://localhost:8282/protocol` accordingly. - -### Consumer connector - -The consumer is the one "requesting" the data and providing a destination for it, i.e. a directory into which the -provider can copy the requested file. - -We configure the consumer's API ports in `consumer/config.properties`, so that it does not use the same ports as the -provider. In the config file, we also need to configure the API key authentication, as we're going to use -endpoints from the EDC's management API in this sample and integrated the extension for token-based API -authentication. Therefore, we add the property `edc.api.auth.key` and set it to e.g. `password`. And last, we also need -to configure the consumer's webhook address. We expose the DSP API endpoints on a different port and path than other -endpoints, so the property `edc.dsp.callback.address` is adjusted to match the DSP API port. - -## Run the sample - -Running this sample consists of multiple steps, that are executed one by one. - -### 1. Build and start the connectors - -The first step to running this sample is building and starting both the provider and the consumer connector. This is -done the same way as in the previous samples. - -```bash -./gradlew transfer:transfer-01-file-transfer:file-transfer-consumer:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-consumer/build/libs/consumer.jar -# in another terminal window: -./gradlew transfer:transfer-01-file-transfer:file-transfer-provider:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-provider/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-provider/build/libs/provider.jar -```` - -Assuming you didn't change the ports in config files, the consumer will listen on the ports `9191`, `9192` -(management API) and `9292` (PROTOCOL API) and the provider will listen on the ports `8181`, `8182` -(management API) and `8282` (PROTOCOL API). - -### 2. Initiate a contract negotiation - -In order to request any data, a contract agreement has to be negotiated between provider and consumer. The provider -offers all of their assets in the form of contract offers, which are the basis for such a negotiation. In the -`transfer-file-local` extension, we've added a contract definition (from which contract offers can be created) for the -file, but the consumer has yet to accept this offer. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also send counter offers in -addition to just confirming or declining an offer. - -In order to trigger the negotiation, we use a management API endpoint. We set our contract offer in the request -body. The contract offer is prepared in [contractoffer.json](contractoffer.json) and can be used as is. In a real -scenario, a potential consumer would first need to request a description of the provider's offers in order to get the -provider's contract offer. - -> Note, that we need to specify the `X-Api-Key` header, as we integrated token-based API authentication. The value -of the header has to match the value of the `edc.api.auth.key` property in the consumer's `config.properties`. - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" -``` - -In the response we'll get a UUID that we can use to get the contract agreement negotiated between provider and consumer. - -Sample output: - -```json -{"@id":"5a6b7e22-dc7d-4135-bc98-4cc5fd1dd1ed"} -``` - -### 3. Look up the contract agreement ID - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. This UUID is the ID of -the ongoing contract negotiation between consumer and provider. The negotiation sequence between provider and consumer -is executed asynchronously in the background by a state machine. Once both provider and consumer either reach the -`confirmed` or the `declined` state, the negotiation is finished. We can now use the UUID to check the current status -of the negotiation using an endpoint on the consumer side. Again, we use the `X-Api-Key` header with the same value -that's set in our consumer's `config.properties`. - -```bash -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" -``` - -This will return information about the negotiation, which contains e.g. the current state of the negotiation and, if the -negotiation has been completed successfully, the ID of a contract agreement. We can now use this agreement to request -the file. So we copy and store the agreement ID for the next step. - -Sample output: - -```json -{ - ... - "edc:contractAgreementId":"1:test-document:fb80be14-8e09-4e50-b65d-c269bc1f16d0", - "edc:state":"FINALIZED", - ... -} -``` - -If you see an output similar to the following, the negotiation has not yet been completed. In this case, -just wait for a moment and call the endpoint again. - -```json -{ - ... - "edc:state": "REQUESTED", - "edc:contractAgreementId": null, - ... -} -``` - -### 4. Request the file - -Now that we have a contract agreement, we can finally request the file. In the request body we need to specify -which asset we want transferred, the ID of the contract agreement, the address of the provider connector and where -we want the file transferred. The request body is prepared in [filetransfer.json](filetransfer.json). Before executing -the request, insert the contract agreement ID from the previous step and adjust the destination location for the file -transfer. Then run: - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" -``` - -Again, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and performed asynchronously. - -Sample output: - -```json -{"@id":"deeed974-8a43-4fd5-93ad-e1b8c26bfa44"} -``` - -Since transferring a file does not require any resource provisioning on either side, the transfer will be very quick and -most likely already done by the time you read the UUID. - ---- - -You can also check the logs of the connectors to see that the transfer has been completed: - -Consumer side: - -```bash -DEBUG 2022-05-03T10:37:59.599642754 Starting transfer for asset asset-id -DEBUG 2022-05-03T10:37:59.6071347 Transfer process initialised f925131b-d61e-48b9-aa15-0f5e2e749064 -DEBUG 2022-05-03T10:38:01.230902645 TransferProcessManager: Sending process f925131b-d61e-48b9-aa15-0f5e2e749064 request to http://localhost:8282/protocol -DEBUG 2022-05-03T10:38:01.260916372 Response received from connector. Status 200 -DEBUG 2022-05-03T10:38:01.285641788 TransferProcessManager: Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now REQUESTED -DEBUG 2022-05-03T10:38:06.246094874 Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now IN_PROGRESS -DEBUG 2022-05-03T10:38:06.246755642 Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now COMPLETED -``` - -### 5. See transferred file - -After the file transfer is completed, we can check the destination path specified in the request for the file. Here, -we'll now find a file with the same content as the original file offered by the provider. - ---- - -[Next Chapter](../transfer-02-file-transfer-listener/README.md) diff --git a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts deleted file mode 100644 index 5d9ff687..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.api.observability) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) - -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties b/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties deleted file mode 100644 index 1655aa4f..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties +++ /dev/null @@ -1,11 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -web.http.protocol.port=9292 -web.http.protocol.path=/protocol - -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9292/protocol -edc.participant.id=consumer -edc.ids.id=urn:connector:consumer diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts deleted file mode 100644 index d1a7776c..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.api.client) - implementation(libs.edc.control.plane.api) - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.api.observability) - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - implementation(libs.edc.dsp) - - implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("provider.jar") -} diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties b/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties deleted file mode 100644 index 2383072c..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties +++ /dev/null @@ -1,13 +0,0 @@ -web.http.port=8181 -web.http.path=/api -web.http.management.port=8182 -web.http.management.path=/management -web.http.protocol.port=8282 -web.http.protocol.path=/protocol -web.http.control.port=8283 -web.http.control.path=/control -edc.samples.transfer.01.asset.path=/path/to/file -edc.dsp.callback.address=http://localhost:8282/protocol -edc.participant.id=provider -edc.ids.id=urn:connector:provider -edc.control.endpoint=http://localhost:8283/control diff --git a/transfer/transfer-01-file-transfer/filetransfer.json b/transfer/transfer-01-file-transfer/filetransfer.json deleted file mode 100644 index 035bbb49..00000000 --- a/transfer/transfer-01-file-transfer/filetransfer.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "dataDestination": { - "type": "File", - "path": "{path to destination file}", - "keyName": "keyName" - }, - "protocol": "dataspace-protocol-http", - "assetId": "test-document", - "contractId": "{agreement ID}", - "connectorId": "provider", - "connectorAddress": "http://localhost:8282/protocol", - "privateProperties": {} -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts b/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts deleted file mode 100644 index 9051a9a9..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` - id("application") -} - -dependencies { - api(libs.edc.control.plane.spi) - api(libs.edc.data.plane.spi) - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.core) - implementation(libs.edc.data.plane.util) - implementation(libs.edc.data.plane.client) - implementation(libs.edc.data.plane.selector.client) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.transfer.data.plane) - implementation(libs.opentelemetry.annotations) -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java deleted file mode 100644 index 1fb0b4dd..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import io.opentelemetry.extension.annotations.WithSpan; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; - -import java.io.File; -import java.io.FileOutputStream; -import java.util.List; -import java.util.Objects; - -import static java.lang.String.format; -import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure.Reason.GENERAL_ERROR; - -class FileTransferDataSink extends ParallelSink { - private File file; - - @WithSpan - @Override - protected StreamResult transferParts(List parts) { - for (DataSource.Part part : parts) { - var fileName = part.name(); - try (var input = part.openStream()) { - try (var output = new FileOutputStream(file)) { - try { - input.transferTo(output); - } catch (Exception e) { - return getTransferResult(e, "Error transferring file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error creating file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error reading file %s", fileName); - } - } - return StreamResult.success(); - } - - private StreamResult getTransferResult(Exception e, String logMessage, Object... args) { - var message = format(logMessage, args); - monitor.severe(message, e); - return StreamResult.failure(new StreamFailure(List.of(message), GENERAL_ERROR)); - } - - public static class Builder extends ParallelSink.Builder { - - public static Builder newInstance() { - return new Builder(); - } - - public Builder file(File file) { - sink.file = file; - return this; - } - - @Override - protected void validate() { - Objects.requireNonNull(sink.file, "file"); - } - - private Builder() { - super(new FileTransferDataSink()); - } - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java deleted file mode 100644 index e7befc2a..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.util.concurrent.ExecutorService; - -class FileTransferDataSinkFactory implements DataSinkFactory { - private final Monitor monitor; - private final ExecutorService executorService; - private final int partitionSize; - - FileTransferDataSinkFactory(Monitor monitor, ExecutorService executorService, int partitionSize) { - this.monitor = monitor; - this.executorService = executorService; - this.partitionSize = partitionSize; - } - - @Override - public boolean canHandle(DataFlowRequest request) { - return "file".equalsIgnoreCase(request.getDestinationDataAddress().getType()); - } - - @Override - public DataSink createSink(DataFlowRequest request) { - var destination = request.getDestinationDataAddress(); - - // verify destination path - var path = destination.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - var destinationFile = new File(path.replaceAll("\\.", ".").replaceAll("/", "/")); - - return FileTransferDataSink.Builder.newInstance() - .file(destinationFile) - .requestId(request.getId()) - .partitionSize(partitionSize) - .executorService(executorService) - .monitor(monitor) - .build(); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - return Result.success(); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java deleted file mode 100644 index f946039f..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.spi.EdcException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.util.stream.Stream; - -class FileTransferDataSource implements DataSource { - - private final File file; - - FileTransferDataSource(File file) { - this.file = file; - } - - @Override - public StreamResult> openPartStream() { - var part = new Part() { - @Override - public String name() { - return file.getName(); - } - - @Override - public InputStream openStream() { - try { - return new FileInputStream(file); - } catch (FileNotFoundException e) { - throw new EdcException(e); - } - } - }; - return StreamResult.success(Stream.of(part)); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java deleted file mode 100644 index 3712b9ce..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; - -import java.io.File; - -class FileTransferDataSourceFactory implements DataSourceFactory { - @Override - public boolean canHandle(DataFlowRequest dataRequest) { - return "file".equalsIgnoreCase(dataRequest.getSourceDataAddress().getType()); - } - - @Override - public DataSource createSource(DataFlowRequest request) { - var source = getFile(request); - return new FileTransferDataSource(source); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - var source = getFile(request); - if (!source.exists()) { - return Result.failure("Source file " + source.getName() + " does not exist!"); - } - - return Result.success(); - } - - @NotNull - private File getFile(DataFlowRequest request) { - var dataAddress = request.getSourceDataAddress(); - // verify source path - var sourceFileName = dataAddress.getStringProperty("filename"); - var path = dataAddress.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - sourceFileName = sourceFileName.replaceAll("\\.", ".").replaceAll("/", "/"); - path = path.replaceAll("\\.", ".").replaceAll("/", "/"); - return new File(path, sourceFileName); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java deleted file mode 100644 index 23d67e99..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2021 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; -import org.eclipse.edc.connector.contract.spi.types.offer.ContractDefinition; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.connector.policy.spi.PolicyDefinition; -import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.policy.model.PolicyType; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.asset.AssetIndex; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.asset.Asset; - -import java.nio.file.Path; - -import static org.eclipse.edc.spi.query.Criterion.criterion; - -public class FileTransferExtension implements ServiceExtension { - - public static final String USE_POLICY = "use-eu"; - private static final String EDC_ASSET_PATH = "edc.samples.transfer.01.asset.path"; - @Inject - private ContractDefinitionStore contractStore; - @Inject - private AssetIndex assetIndex; - @Inject - private PipelineService pipelineService; - @Inject - private DataTransferExecutorServiceContainer executorContainer; - @Inject - private PolicyDefinitionStore policyStore; - - @Override - public void initialize(ServiceExtensionContext context) { - var monitor = context.getMonitor(); - - var sourceFactory = new FileTransferDataSourceFactory(); - pipelineService.registerFactory(sourceFactory); - - var sinkFactory = new FileTransferDataSinkFactory(monitor, executorContainer.getExecutorService(), 5); - pipelineService.registerFactory(sinkFactory); - - var policy = createPolicy(); - policyStore.create(policy); - - registerDataEntries(context); - registerContractDefinition(policy.getUid()); - - context.getMonitor().info("File Transfer Extension initialized!"); - } - - private PolicyDefinition createPolicy() { - return PolicyDefinition.Builder.newInstance() - .id(USE_POLICY) - .policy(Policy.Builder.newInstance() - .type(PolicyType.SET) - .build()) - .build(); - } - - private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); - var assetPath = Path.of(assetPathSetting); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("path", assetPath.getParent().toString()) - .property("filename", assetPath.getFileName().toString()) - .build(); - - var assetId = "test-document"; - var asset = Asset.Builder.newInstance() - .id(assetId) - .dataAddress(dataAddress) - .build(); - - assetIndex.create(asset); - } - - private void registerContractDefinition(String uid) { - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(uid) - .contractPolicyId(uid) - .assetsSelectorCriterion(criterion(Asset.PROPERTY_ID, "=", "test-document")) - .build(); - - contractStore.save(contractDefinition); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 5cc8c302..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.sample.extension.api.FileTransferExtension \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/README.md b/transfer/transfer-01-negotiation/README.md new file mode 100644 index 00000000..885d9a6e --- /dev/null +++ b/transfer/transfer-01-negotiation/README.md @@ -0,0 +1,242 @@ +# Implement a negotiation workflow between two connectors + + +Before two connectors can exchange actual data, negotiation has to take place. + +The final goal of this example is to showcase the negotiation workflow between two connectors so that +the actual data transfer can take place. The actual data transfer will be part of the next chapters. + +You will: + +* Create an asset on the provider (the asset will be the data to be shared) +* Create an access policy on the provider (the policy will define the access right to the data) +* Create a contract definition on the provider + +At this step, the consumer should be able to fetch the catalog from the provider and to see the +contract offer generated from the resources that have been created. + +Once the catalog is available, to access the data, the consumer should follow the following steps: + +* Perform a contract negotiation with the provider + +> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single +> asset. This will be deleted after the provider shutdown. + +The provider connector is the one managing assets and respective access policies for which a contract negotiation +can be executed while the consumer is the one "requesting" assets and initiating contract negotiations. + +## Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +The following steps assume your provider and consumer connector are still up and running. +If not, re-run them as described in the [Prerequisites](../transfer-00-prerequisites/README.md) chapter. + +### 1. Create an Asset on the provider side + +The provider connector needs to transfer a file to the location specified by the consumer connector +when the data are requested. In order to offer any data, the provider must maintain an internal list +of resources offered, through a contract offer, the so-called "catalog". + +The following [request](resources/create-asset.json) creates an asset on the provider connector. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ + -s | jq +``` + +> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for +> the purpose of this example. It will be the data that the consumer will pull on the sample +> execution. + +Additional properties on `HttpData` can be used to allow consumers to enrich the data request: + +- `proxyPath`: allows specifying additional path segments. +- `proxyQueryParams`: allows specifying query params. +- `proxyBody`: allows attaching a body. +- `proxyMethod`: allows specifying the Http Method (default `GET`) + +### 2. Create a Policy on the provider + +In order to manage the accessibility rules of an asset, it is essential to create a policy. However, +to keep things simple, we will choose a policy that gives direct access to all the assets that are +associated within the contract definitions. +This means that the consumer connector can request any asset from the provider connector. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ + -s | jq +``` + +### 3. Create a contract definition on Provider + +To ensure an exchange between providers and consumers, the supplier must create a contract offer for +the good, on the basis of which a contract agreement can be negotiated. The contract definition +associates policies to a selection of assets to generate the contract offers that will be put in the +catalog. In this case, the selection is empty, so every asset is attached to these policies + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ + -s | jq + +``` + +Sample output: + +```json +{ + ... + "@id": "1", + "edc:createdAt": 1674578184023, + ... +} +``` + +### 4. How to fetch catalog on consumer side + +In order to offer any data, the consumer can fetch the catalog from the provider, that will contain +all the contract offers available for negotiation. In our case, it will contain a single contract +offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following +request: + +```bash +curl -X POST "http://localhost:29193/management/v2/catalog/request" \ + -H 'Content-Type: application/json' \ + -d @transfer/transfer-01-negotiation/resources/fetch-catalog.json -s | jq +``` + +Sample output: + +```json +{ + "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", + "@type": "dcat:Catalog", + "dcat:dataset": { + "@id": "assetId", + "@type": "dcat:Dataset", + "odrl:hasPolicy": { + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "odrl:Set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "assetId" + }, + "dcat:distribution": [ + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpProxy" + }, + "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" + }, + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpData" + }, + "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" + } + ], + "edc:name": "product description", + "edc:id": "assetId", + "edc:contenttype": "application/json" + }, + "dcat:service": { + "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", + "@type": "dcat:DataService", + "dct:terms": "connector", + "dct:endpointUrl": "http://localhost:19194/protocol" + }, + "edc:participantId": "anonymous", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` + +### 5. Negotiate a contract + +In order to request any data, a contract gets negotiated, and an agreement is resulting has to be +negotiated between providers and consumers. + +The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence +looks as follows: + +1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the + provider's offer!__) +2. Provider validates the received offer against its own offer +3. Provider either sends an agreement or a rejection, depending on the validation result +4. In case of successful validation, provider and consumer store the received agreement for later + reference + +Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also +send counter offers in addition to just confirming or declining an offer. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ + -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ + -s | jq +``` + +Sample output: + +```json +{ + ... + "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", + "edc:createdAt": 1685525281848, + ... +} +``` + +### 6. Getting the contract agreement id + +After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. +This UUID is the ID of the ongoing contract negotiation between consumer and provider. The +negotiation sequence between provider and consumer is executed asynchronously in the background by a +state machine. Once both provider and consumer either reach the `confirmed` or the `declined` +state, the negotiation is finished. We can now use the UUID to check the current status of the +negotiation using an endpoint on the consumer side. + +```bash +curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ + --header 'Content-Type: application/json' \ + -s | jq +``` + +Sample output: + +```json +{ + "@type": "edc:ContractNegotiationDto", + "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", + "edc:type": "CONSUMER", + "edc:protocol": "dataspace-protocol-http", + "edc:state": "FINALIZED", + "edc:counterPartyAddress": "http://localhost:19194/protocol", + "edc:callbackAddresses": [], + "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", <--------- + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` +At this point contract negotiation has been successfully completed. +The connectors are now ready to enter the data transfer phase. + +Note down the `contractAgreementId`. You will need it in the next chapters. + +[Next Chapter](../transfer-02-consumer-pull/README.md) \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-asset.json b/transfer/transfer-01-negotiation/resources/create-asset.json new file mode 100644 index 00000000..b6abaf3f --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-asset.json @@ -0,0 +1,18 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "asset": { + "@id": "assetId", + "properties": { + "name": "product description", + "contenttype": "application/json" + } + }, + "dataAddress": { + "type": "HttpData", + "name": "Test asset", + "baseUrl": "https://jsonplaceholder.typicode.com/users", + "proxyPath": "true" + } +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-contract-definition.json b/transfer/transfer-01-negotiation/resources/create-contract-definition.json new file mode 100644 index 00000000..4094d276 --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-contract-definition.json @@ -0,0 +1,9 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "1", + "accessPolicyId": "aPolicy", + "contractPolicyId": "aPolicy", + "assetsSelector": [] +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-policy.json b/transfer/transfer-01-negotiation/resources/create-policy.json new file mode 100644 index 00000000..e62eb3cc --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-policy.json @@ -0,0 +1,13 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "aPolicy", + "policy": { + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [] + } +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/fetch-catalog.json b/transfer/transfer-01-negotiation/resources/fetch-catalog.json new file mode 100644 index 00000000..74994a15 --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "providerUrl": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-01-negotiation/resources/negotiate-contract.json similarity index 57% rename from transfer/transfer-04-open-telemetry/contractoffer.json rename to transfer/transfer-01-negotiation/resources/negotiate-contract.json index 5baecd25..33a46d2a 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-01-negotiation/resources/negotiate-contract.json @@ -5,20 +5,20 @@ }, "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", "consumerId": "consumer", "providerId": "provider", - "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", + "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "assetId": "assetId", "policy": { - "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", - "@type": "set", + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "Set", "odrl:permission": [], "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "test-document" + "odrl:target": "assetId" } } -} +} \ No newline at end of file diff --git a/transfer/transfer-02-consumer-pull/README.md b/transfer/transfer-02-consumer-pull/README.md new file mode 100644 index 00000000..c1e41227 --- /dev/null +++ b/transfer/transfer-02-consumer-pull/README.md @@ -0,0 +1,131 @@ +# Implement a simple "Consumer Pull" Http transfer flow + +The purpose of this sample is to show a data exchange between 2 connectors, one representing the +data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find +more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +This sample consists of the following steps: + +* Perform a file transfer initiated by the consumer +* The provider will send an EndpointDataReference to the consumer +* The consumer will call the endpoint and fetch the data + +## Prerequisites + +The following steps assume your provider and consumer connectors are still up and running and contract +negotiation has taken place successfully. +If not, re-visit the [Prerequisites](../transfer-00-prerequisites/README.md) +and [Negotiation](../transfer-01-negotiation/README.md) chapters. + +# Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 1. Start a http server + +As a pre-requisite, you need to have a http server that runs on port 4000 and logs all the incoming requests, it will +be mandatory to get the EndpointDataReference that will be used to get the data. + +```bash +docker build -t http-request-logger util/http-request-logger +docker run -p 4000:4000 http-request-logger +``` + +### 2. Start the transfer + +In the [request body](resources/start-transfer.json), we need to specify which asset we want transferred, the ID of the contract agreement, the address of the +provider connector and where we want the file transferred. +Before executing the request, insert the `contractAgreementId` from the previous chapter. Then run: + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-02-consumer-pull/resources/start-transfer.json \ + -s | jq + +``` + +> the "HttpProxy" method is used for the consumer pull method, and it means that it will be up to +> the consumer to request the data to the provider and that the request will be a proxy for the +> datasource + +Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( +process id) created on the consumer +side, because like the contract negotiation, the data transfer is handled in a state machine and +performed asynchronously. + +Sample output: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:createdAt": 1674078357807, + ... +} +``` + +### 3. Check the transfer status + +Due to the nature of the transfer, it will be very fast and most likely already done by the time you +read the UUID. + +```bash +curl http://localhost:29193/management/v2/transferprocesses/ +``` + +You should see the Transfer Process in `STARTED` state: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:state": "STARTED", + ... +} + +``` + +> Note that for the consumer pull scenario the TP will stay in STARTED state after the data has been transferred successfully. +> It might get eventually get shifted to TERMINATED or DEPROVISIONED by other resources, but this is not scope of this sample. + +### 4. Check the data + +At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed +to get the data from the provider: + +```json +{ + "id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "endpoint": "http://localhost:29291/public/", + "authKey": "Authorization", + "authCode": "eyJhbGciOiJSUzI1NiJ9.eyJkYWQiOiJ7XCJwcm9wZXJ0aWVzXCI6e1wiYXV0aEtleVwiOlwiQXV0aG9yaXphdGlvblwiLFwiYmFzZVVybFwiOlwiaHR0cDpcL1wvbG9jYWxob3N0OjE5MjkxXC9wdWJsaWNcL1wiLFwiYXV0aENvZGVcIjpcImV5SmhiR2NpT2lKU1V6STFOaUo5LmV5SmtZV1FpT2lKN1hDSndjbTl3WlhKMGFXVnpYQ0k2ZTF3aVltRnpaVlZ5YkZ3aU9sd2lhSFIwY0hNNlhDOWNMMnB6YjI1d2JHRmpaV2h2YkdSbGNpNTBlWEJwWTI5a1pTNWpiMjFjTDNWelpYSnpYQ0lzWENKdVlXMWxYQ0k2WENKVVpYTjBJR0Z6YzJWMFhDSXNYQ0owZVhCbFhDSTZYQ0pJZEhSd1JHRjBZVndpZlgwaUxDSmxlSEFpT2pFMk56UTFPRGcwTWprc0ltTnBaQ0k2SWpFNk1XVTBOemc1TldZdE9UQXlOUzAwT1dVeExUazNNV1F0WldJNE5qVmpNemhrTlRRd0luMC5ITFJ6SFBkT2IxTVdWeWdYZi15a0NEMHZkU3NwUXlMclFOelFZckw5eU1tQjBzQThwMHFGYWV0ZjBYZHNHMG1HOFFNNUl5NlFtNVU3QnJFOUwxSE5UMktoaHFJZ1U2d3JuMVhGVUhtOERyb2dSemxuUkRlTU9ZMXowcDB6T2MwNGNDeFJWOEZoemo4UnVRVXVFODYwUzhqbU4wZk5sZHZWNlFpUVFYdy00QmRTQjNGYWJ1TmFUcFh6bDU1QV9SR2hNUGphS2w3RGsycXpJZ0ozMkhIdGIyQzhhZGJCY1pmRk12aEM2anZ2U1FieTRlZXU0OU1hclEydElJVmFRS1B4ajhYVnI3ZFFkYV95MUE4anNpekNjeWxyU3ljRklYRUV3eHh6Rm5XWmczV2htSUxPUFJmTzhna2RtemlnaXRlRjVEcmhnNjZJZzJPR0Eza2dBTUxtc3dcIixcInByb3h5TWV0aG9kXCI6XCJ0cnVlXCIsXCJwcm94eVF1ZXJ5UGFyYW1zXCI6XCJ0cnVlXCIsXCJwcm94eUJvZHlcIjpcInRydWVcIixcInR5cGVcIjpcIkh0dHBEYXRhXCIsXCJwcm94eVBhdGhcIjpcInRydWVcIn19IiwiZXhwIjoxNjc0NTg4NDI5LCJjaWQiOiIxOjFlNDc4OTVmLTkwMjUtNDllMS05NzFkLWViODY1YzM4ZDU0MCJ9.WhbTzERmM75mNMUG2Sh-8ZW6uDQCus_5uJPvGjAX16Ucc-2rDcOhAxrHjR_AAV4zWjKBHxQhYk2o9jD-9OiYb8Urv8vN4WtYFhxJ09A0V2c6lB1ouuPyCA_qKqJEWryTbturht4vf7W72P37ERo_HwlObOuJMq9CS4swA0GBqWupZHAnF-uPIQckaS9vLybJ-gqEhGxSnY4QAZ9-iwSUhkrH8zY2GCDkzAWIPmvtvRhAs9NqVkoUswG-ez1SUw5bKF0hn2OXv_KhfR8VsKKYUbKDQf5Wagk7rumlYbXMPNAEEagI4R0xiwKWVTfwwZPy_pYnHE7b4GQECz3NjhgdIw", + "properties": { + "cid": "1:1e47895f-9025-49e1-971d-eb865c38d540" + } +} +``` + +Once this json is read, use a tool like postman or curl to execute the following query, to read the +data + +```bash +curl --location --request GET 'http://localhost:29291/public/' --header 'Authorization: ' +``` + +At the end, and to be sure that you correctly achieved the pull, you can check if the data you get +is the same as the one you can get at https://jsonplaceholder.typicode.com/users + + +Since we configured the `HttpData` with `proxyPath`, we could also ask for a specific user with: + +```bash +curl --location --request GET 'http://localhost:29291/public/1' --header 'Authorization: ' +``` + +And the data returned will be the same as in https://jsonplaceholder.typicode.com/users/1 + +Your first data transfer has been completed successfully. +Continue with the [next chapter](../transfer-03-provider-push/README.md) to run through a "provider push" scenario. \ No newline at end of file diff --git a/transfer/transfer-02-consumer-pull/resources/start-transfer.json b/transfer/transfer-02-consumer-pull/resources/start-transfer.json new file mode 100644 index 00000000..e28fc9c5 --- /dev/null +++ b/transfer/transfer-02-consumer-pull/resources/start-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} \ No newline at end of file diff --git a/transfer/transfer-02-file-transfer-listener/README.md b/transfer/transfer-02-file-transfer-listener/README.md deleted file mode 100644 index 0bc11666..00000000 --- a/transfer/transfer-02-file-transfer-listener/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Implement a simple transfer listener - -In this sample, we build upon the [file transfer sample](../transfer-01-file-transfer/README.md) to add functionality -to react to transfer completion on the consumer connector side. - -We will use the provider from the [file transfer sample](../transfer-01-file-transfer/README.md), and the consumer -built on the consumer from that sample, with a transfer process listener added. - -Also, in order to keep things organized, the code in this example has been separated into several Java modules: - -- `file-transfer-listener-consumer`: this is where the extension definition and dependencies reside for the consumer connector -- `listener`: contains the `TransferProcessListener` implementation - -## Create the listener - -A TransferProcessListener may define methods that are invoked after a transfer changes state, for example, to notify an -external application on the consumer side after data has been produced (i.e. the transfer moves to the completed state). - -```java -// in TransferListenerExtension.java - @Override - public void initialize(ServiceExtensionContext context) { - // ... - var transferProcessObservable = context.getService(TransferProcessObservable.class); - transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); - } -``` - -```java -public class MarkerFileCreator implements TransferProcessListener { - - /** - * Callback invoked by the EDC framework when a transfer has completed. - * - * @param process - */ - @Override - public void completed(final TransferProcess process) { - // ... - } -} -``` - -## Perform a file transfer - -Let's rebuild and run them both: - -```bash -./gradlew transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer:build -java -Dedc.fs.config=transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties -jar transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build/libs/consumer.jar -# in another terminal window: -./gradlew transfer:transfer-01-file-transfer:file-transfer-provider:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-provider/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-provider/build/libs/provider.jar -```` - -Assuming you didn't change the config files, the consumer will expose management api on port `9192` and the custom -api endpoints on port `9191` and the provider will listen on port `8181`. -Open another terminal window (or any REST client of your choice) and execute the following REST requests like in the -previous sample: - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" -``` - -> **Replace `{UUID}` in the second request with the UUID received as the response to the first request!** -> -> **In `transfer/transfer-01-file-transfer/filetransfer.json`: Copy the contract agreement's ID from the second response, -> substitute it for `{agreement ID}` in the last request JSON body and adjust the `dataDestination.properties.path` to match your local dev machine!** - -- `curl` will return the ID of the transfer process on the consumer connector. - -The consumer should spew out logs similar to: - -```bash -DEBUG 2022-04-14T16:23:13.4042547 Starting transfer for asset test-document -DEBUG 2022-04-14T16:23:13.4072776 Transfer process initialised 6804ed96-298e-4992-b72d-2366d97cf7a6 -DEBUG 2022-04-14T16:23:13.8801678 TransferProcessManager: Sending process 6804ed96-298e-4992-b72d-2366d97cf7a6 request to http://localhost:8282/protocol -DEBUG 2022-04-14T16:23:13.9341802 TransferProcessManager: Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now REQUESTED -DEBUG 2022-04-14T16:23:18.9048494 Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now IN_PROGRESS -DEBUG 2022-04-14T16:23:18.9048494 Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now COMPLETED -INFO 2022-04-14T16:23:18.9048494 Transfer Listener successfully wrote file C:\Users\pechande\dev\coding\EDC\marker.txt -``` - -Then check `/path/on/yourmachine`, which should now contain a file named `marker.txt` in addition to the file defined -in `dataDestination.properties.path` in `transfer/transfer-01-file-transfer/filetransfer.json`. - ---- - -[Previous Chapter](../transfer-01-file-transfer/README.md) | [Next Chapter](../transfer-03-modify-transferprocess/README.md) diff --git a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts b/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts deleted file mode 100644 index e8e9e82c..00000000 --- a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.http) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) - - implementation(project(":transfer:transfer-02-file-transfer-listener:listener")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - exclude("**/pom.properties", "**/pom.xml") - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties b/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties deleted file mode 100644 index e162bff0..00000000 --- a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties +++ /dev/null @@ -1,9 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -web.http.protocol.port=9292 -web.http.protocol.path=/protocol - -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9292/protocol diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 97c66199..00000000 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.sample.extension.listener.TransferListenerExtension \ No newline at end of file diff --git a/transfer/transfer-03-modify-transferprocess/README.md b/transfer/transfer-03-modify-transferprocess/README.md deleted file mode 100644 index 1f1c9845..00000000 --- a/transfer/transfer-03-modify-transferprocess/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Modify a TransferProcess - -In the last transfer samples (`transfer-01` and `transfer-02`) we saw how data can be transferred easily, what a -`TransferProcess` is and how to react to it easily through the listener. This sample will show how `TransferProcess` -objects can be modified externally in a thread-safe and consistent way. - -## Problem statement - -The `TransferProcessManager` (TPM), which is the central state machine handling transfer processes, follows this basic -operational pattern: - -1. take transfer process (TP) out of `TransferProcessStore` (TPS) -2. take appropriate action, e.g. provision or deprovision resources -3. update state of TP -4. put back into TPS - -All those steps happen in a non-atomic way, so when a TP currently processed by the TPM is modified on another thread, -there is a strong possibility that that change will get overwritten or worse, may cause the state machine to be in an -illegal state. - -A common pattern would be some sort of watchdog, where TPs that have not advanced their state in a given amount of time -are automatically cancelled or errored out. The following code snippet shows a typical TPM state transition: - -```java -// get out of store - var tpList = store.nextForState(IN_PROGRESS.code(),batchSize); -// take appropriate action, e.g. check if complete -var statusChecker=...; -foreach(var tp:tpList){ - if(statusChecker.isComplete()){ - //update state - tp.transitionComplete(); - // put back into TPS - store.update(tp); - } -} -``` -and then consider a watchdog that runs on another thread and fires every X minutes -```java - -private void watchDog(){ - var longRunningTpList = store.nextForState(IN_PROGRESS.code(), 5); - // filter list based on last state update - var longRunningTpList = /*filter expression*/; - for(var tp : longRunningTpList){ - tp.transitionError("timeout"); - store.update(tp); - } -} -``` - -Now the problem becomes apparent when the `watchDog()` fires exactly here: -```java -//... - if(statusChecker.isComplete()){ - - // |<-- watchDog() fires here! - - //update state - tp.transitionComplete(); - // ... - } -``` - -then the TP would first go to the `ERROR` state, but then immediately to the `COMPLETED` state, because the TPM and the watchdog -have different object references to the same TP. We essentially have a race condition at our hands, resulting in the TP never -"erroring out". - -## About this sample -Please note that this sample does _not actually transfer anything_, it merely shows how to modify a transfer process -outside the main state machine. - -Modules: -- `simulator`: used to insert a dummy transfer process, that never completes to simulate the use of a watchdog -- `watchdog`: spins up a periodic task that checks for timed-out TPs and sets them to `ERROR` -- `consumer`: the build configuration - -In order to run the sample, enter the following commands in a shell: - -```bash -./gradlew transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer:build -java -Dedc.fs.config=transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties -jar transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build/libs/consumer.jar -``` - ---- - -[Previous Chapter](../transfer-02-file-transfer-listener/README.md) | [Next Chapter](../transfer-04-open-telemetry/README.md) diff --git a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts deleted file mode 100644 index 497de949..00000000 --- a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(project(":transfer:transfer-03-modify-transferprocess:watchdog")) - implementation(project(":transfer:transfer-03-modify-transferprocess:simulator")) - - implementation(libs.edc.control.plane.core) - implementation(libs.edc.http) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties b/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties deleted file mode 100644 index 863cdab3..00000000 --- a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties +++ /dev/null @@ -1,6 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9191/protocol diff --git a/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts deleted file mode 100644 index 66cd3278..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` -} - -dependencies { - implementation(libs.edc.control.plane.core) -} diff --git a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java b/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java deleted file mode 100644 index eb071a9d..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedDataDestinationResource; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.jetbrains.annotations.NotNull; - -import java.time.Clock; -import java.time.Duration; -import java.util.Timer; -import java.util.TimerTask; - -public class TransferSimulationExtension implements ServiceExtension { - - public static final String TEST_TYPE = "test-type"; - private static final long ALMOST_TEN_MINUTES = Duration.ofMinutes(9).plusSeconds(55).toMillis(); - @Inject - private TransferProcessStore store; - - @Inject - private Clock clock; - - @Override - public void initialize(ServiceExtensionContext context) { - //Insert a test TP after a delay to simulate a zombie transfer - new Timer().schedule( - new TimerTask() { - @Override - public void run() { - var tp = TransferProcess.Builder.newInstance() - .id("tp-sample-transfer-03") - .dataRequest(getRequest()) - .state(TransferProcessStates.STARTED.code()) - .stateTimestamp(clock.millis() - ALMOST_TEN_MINUTES) - .build(); - tp.addProvisionedResource(createDummyResource()); - - context.getMonitor().info("Insert Dummy TransferProcess"); - store.save(tp); - } - }, - 5000 - ); - } - - @NotNull - private ProvisionedDataDestinationResource createDummyResource() { - return new ProvisionedDataDestinationResource() { - }; - } - - private DataRequest getRequest() { - return DataRequest.Builder.newInstance() - .id("sample-transfer-03-datarequest") - .assetId("assetId") - .contractId("contractId") - .destinationType(TEST_TYPE) - .connectorAddress("http//localhost:9999") - .protocol("dataspace-protocol-http") - .build(); - } - -} diff --git a/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index f8129d56..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2020, 2021 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.sample.extension.transfer.TransferSimulationExtension - diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts deleted file mode 100644 index c472d4ab..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java deleted file mode 100644 index 380e28e3..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.watchdog; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.query.Criterion; - -import java.time.Clock; -import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static java.lang.String.format; -import static java.time.Instant.ofEpochMilli; -import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; - -public class Watchdog { - - private final Monitor monitor; - private final TransferProcessStore store; - private final Clock clock; - private ScheduledExecutorService executor; - - public Watchdog(Monitor monitor, TransferProcessStore store, Clock clock) { - this.monitor = monitor; - this.store = store; - this.clock = clock; - } - - public void start() { - executor = Executors.newSingleThreadScheduledExecutor(); - // run every 10 minutes, no initial delay - executor.scheduleAtFixedRate(this::check, 10, 10, TimeUnit.SECONDS); - } - - public void stop() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private void check() { - monitor.info("Running watchdog"); - - var states = store.nextNotLeased(3, new Criterion("state", "=", STARTED.code())); - states.stream().filter(tp -> isExpired(tp.getStateTimestamp(), Duration.ofSeconds(10))) - .forEach(tp -> { - monitor.info(format("will retire TP with id [%s] due to timeout", tp.getId())); - - tp.transitionTerminating("timeout by watchdog"); - store.save(tp); - }); - } - - private boolean isExpired(long stateTimestamp, Duration maxAge) { - return ofEpochMilli(stateTimestamp).isBefore(clock.instant().minus(maxAge)); - } -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java deleted file mode 100644 index 3c564f14..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.watchdog; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; - -import java.time.Clock; - -public class WatchdogExtension implements ServiceExtension { - - @Inject - private TransferProcessStore store; - - @Inject - private Clock clock; - - private Watchdog watchDog; - - @Override - public void initialize(ServiceExtensionContext context) { - watchDog = new Watchdog(context.getMonitor(), store, clock); - } - - @Override - public void start() { - watchDog.start(); - } - - @Override - public void shutdown() { - watchDog.stop(); - } -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 72409f33..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2020, 2021 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.sample.extension.watchdog.WatchdogExtension - diff --git a/transfer/transfer-03-provider-push/README.md b/transfer/transfer-03-provider-push/README.md new file mode 100644 index 00000000..07277efa --- /dev/null +++ b/transfer/transfer-03-provider-push/README.md @@ -0,0 +1,69 @@ +# Implement a simple "Provider Push" Http transfer flow + +This sample demonstrates the "provider push" use case that you can find more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +This samples consists of: + +* Performing a file transfer + * The consumer will initiate a file transfer + * Provider Control Plane retrieves the DataAddress of the actual data source and creates a + DataFlowRequest based on the received DataRequest and this data address +* Provider Data Plane fetches data from the actual data source +* Provider Data Plane pushes data to the consumer service + +## Prerequisites + +The following steps assume your provider and consumer connectors are still up and running and contract +negotiation has taken place successfully. Furthermore, the http server should be up as well. +If not, re-visit the [Prerequisites](../transfer-00-prerequisites/README.md) +, [Negotiation](../transfer-01-negotiation/README.md) and [Consumer Pull](../transfer-02-consumer-pull/README.md) chapters. + +# Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 1. Start the transfer + +Before executing the request, modify the [request body](resources/start-transfer.json) by inserting the contract agreement ID +from the [Negotiation](../transfer-01-negotiation/README.md) chapter. +You can re-use the same asset, policies and contract negotiation from before. + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-03-provider-push/resources/start-transfer.json \ + -s | jq +``` +> keep in mind that, to make a transfer with a provider push method, the dataDestination type should +> be any value different from the "HttpProxy". + +Sample output: + +```json + { + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:createdAt": 1674078357807, + ... +} +``` + +### 2. Check the transfer status + +Due to the nature of the transfer, it will be very fast and most likely already done by the time you +read the UUID. + +```bash +curl http://localhost:29193/management/v2/transferprocesses/ +``` + +Notice the transfer COMPLETED state + +### 3. Check the data + +At this step, you can check the data by checking the log of the http server exposed on port 4000, you should see a log +that shows the same data that you can get from https://jsonplaceholder.typicode.com/users. + +[Next Chapter](../transfer-04-event-consumer/README.md) \ No newline at end of file diff --git a/transfer/transfer-03-provider-push/resources/start-transfer.json b/transfer/transfer-03-provider-push/resources/start-transfer.json new file mode 100644 index 00000000..c4636720 --- /dev/null +++ b/transfer/transfer-03-provider-push/resources/start-transfer.json @@ -0,0 +1,15 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpData", + "baseUrl": "http://localhost:4000/api/consumer/store" + } +} \ No newline at end of file diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md new file mode 100644 index 00000000..85952ce9 --- /dev/null +++ b/transfer/transfer-04-event-consumer/README.md @@ -0,0 +1,112 @@ +# Implement a simple event consumer + +In this sample, we build upon the [Consumer Pull](../transfer-02-consumer-pull/README.md) chapter to add functionality +to react to transfer completion on the consumer connector side. + +Also, in order to keep things organized, the code in this example has been separated into several Java modules: + +- `consumer-with-listener`: the consumer connector which will be extended by the event consumer +- `listener`: contains the `TransferProcessListener` implementation which will consume an event + +## Inspect the listener + +A `TransferProcessListener` may define methods that are invoked after a transfer changes state, for example, to notify an +external application on the consumer side after data has been produced (i.e. the transfer moves to the completed state). + +```java +// in TransferListenerExtension.java + @Override + public void initialize(ServiceExtensionContext context) { + // ... + var transferProcessObservable = context.getService(TransferProcessObservable.class); + transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); + } +``` + +The `TransferProcessStartedListener` implements the `TransferProcessListener` interface. +It will consume the transfer `STARTED` event and write a log message. + +```java +public class TransferProcessStartedListener implements TransferProcessListener { + + private final Monitor monitor; + + public TransferProcessStartedListener(Monitor monitor) { + this.monitor = monitor; + } + + /** + * Callback invoked by the EDC framework when a transfer is about to be completed. + * + * @param process the transfer process that is about to be completed. + */ + @Override + public void preStarted(final TransferProcess process) { + monitor.debug("TransferProcessStartedListener received STARTED event"); + // do something meaningful before transfer start + } +} +``` + +## Run the sample + +Assuming your provider connector is still running, we can re-use the existing assets and contract definitions stored on +provider side. If not, set up your assets and contract definitions as described in the [Negotiation](../transfer-01-negotiation/README.md) +chapter. + +### 1. Build & launch the consumer with listener extension + +This consumer connector is based on a different build file, hence a new jar file will be produced. +Make sure to terminate your current consumer connector from the previous chapters. +That way we unblock the ports and can reuse the known configuration files and API calls. + +Run this to build and launch the consumer with listener extension: + +```bash +./gradlew transfer:transfer-04-event-consumer:consumer-with-listener:build +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar +```` + +### 2. Negotiate a new contract + +```bash +curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ + -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ + -s | jq +``` + +### 3. Get the contract agreement id + +```bash +curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ + --header 'Content-Type: application/json' \ + -s | jq +``` + +### 4. Perform a file transfer + +Replace the `contractId` property inside the [request body](../transfer-02-consumer-pull/resources/start-transfer.json) with the contract agreement id from the previous call. +Afterward run: + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-02-consumer-pull/resources/start-transfer.json \ + -s | jq +``` + +### 5. Inspect the logs + +The consumer should spew out logs similar to: + +```bash +DEBUG 2023-10-16T09:29:45.316908 [TransferProcessManagerImpl] TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8fa81 is now in state REQUESTED +DEBUG 2023-10-16T09:29:46.269998 DSP: Incoming TransferStartMessage for class org.eclipse.edc.connector.transfer.spi.types.TransferProcess process: 762b5a0c-43fb-4b8b-8022-669043c8fa81 +DEBUG 2023-10-16T09:29:46.271592 TransferProcessStartedListener received STARTED event <---------------------------- +DEBUG 2023-10-16T09:29:46.27174 TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8fa81 is now in state STARTED +``` + +If you see the `TransferProcessStartedListener received STARTED event` log message, it means that your event consumer has been +configured successfully. + +[Next Chapter](../transfer-05-open-telemetry/README.md) diff --git a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts b/transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts similarity index 86% rename from transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts rename to transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts index 86bc1e6b..805fe5f3 100644 --- a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts +++ b/transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts @@ -18,11 +18,9 @@ plugins { alias(libs.plugins.shadow) } -repositories { - mavenCentral() -} - dependencies { + + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.control.plane.api.client) implementation(libs.edc.dsp) @@ -31,6 +29,7 @@ dependencies { implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.data.plane.selector.api) implementation(libs.edc.data.plane.selector.core) @@ -39,6 +38,8 @@ dependencies { implementation(libs.edc.data.plane.api) implementation(libs.edc.data.plane.core) implementation(libs.edc.data.plane.http) + + implementation(project(":transfer:transfer-04-event-consumer:listener")) } application { @@ -50,6 +51,6 @@ var distZip = tasks.getByName("distZip") tasks.withType { mergeServiceFiles() - archiveFileName.set("push-connector.jar") + archiveFileName.set("connector.jar") dependsOn(distTar, distZip) } diff --git a/transfer/transfer-02-file-transfer-listener/listener/build.gradle.kts b/transfer/transfer-04-event-consumer/listener/build.gradle.kts similarity index 100% rename from transfer/transfer-02-file-transfer-listener/listener/build.gradle.kts rename to transfer/transfer-04-event-consumer/listener/build.gradle.kts diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java similarity index 52% rename from transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java rename to transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java index 4f2714f4..515a9e26 100644 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java +++ b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java @@ -18,17 +18,11 @@ import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.monitor.Monitor; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import static java.lang.String.format; - -public class MarkerFileCreator implements TransferProcessListener { +public class TransferProcessStartedListener implements TransferProcessListener { private final Monitor monitor; - public MarkerFileCreator(Monitor monitor) { + public TransferProcessStartedListener(Monitor monitor) { this.monitor = monitor; } @@ -38,18 +32,8 @@ public MarkerFileCreator(Monitor monitor) { * @param process the transfer process that is about to be completed. */ @Override - public void preCompleted(final TransferProcess process) { - Path path = Path.of(process.getDataRequest().getDataDestination().getStringProperty("path")); - if (!Files.isDirectory(path)) { - path = path.getParent(); - } - path = path.resolve("marker.txt"); - - try { - Files.writeString(path, "Transfer complete"); - monitor.info(format("Transfer Listener successfully wrote file %s", path)); - } catch (IOException e) { - monitor.warning(format("Could not write file %s", path), e); - } + public void preStarted(final TransferProcess process) { + monitor.debug("TransferProcessStartedListener received STARTED event"); + // do something meaningful before transfer start } } diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java similarity index 82% rename from transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java rename to transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java index df2cf070..a2e5f47e 100644 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java +++ b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java @@ -18,14 +18,13 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; -public class TransferListenerExtension implements ServiceExtension { +public class TransferProcessStartedListenerExtension implements ServiceExtension { @Override public void initialize(ServiceExtensionContext context) { var transferProcessObservable = context.getService(TransferProcessObservable.class); var monitor = context.getMonitor(); - - transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); + transferProcessObservable.registerListener(new TransferProcessStartedListener(monitor)); } } diff --git a/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..b53afdfc --- /dev/null +++ b/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.extension.listener.TransferProcessStartedListenerExtension \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml deleted file mode 100644 index b3ad0fca..00000000 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ /dev/null @@ -1,78 +0,0 @@ -version: "3.8" - -services: - - consumer: - build: - context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile - volumes: - - ./:/resources - ports: - - "9191:9191" - - "9192:9192" - environment: - EDC_HOSTNAME: consumer - OTEL_SERVICE_NAME: consumer - OTEL_TRACES_EXPORTER: jaeger - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 - OTEL_METRICS_EXPORTER: prometheus - WEB_HTTP_PORT: 9191 - WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 9192 - WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_PROTOCOL_PORT: 9292 - WEB_HTTP_PROTOCOL_PATH: /protocol - WEB_HTTP_CONTROL_PORT: 9193 - WEB_HTTP_CONTROL_PATH: /control - EDC_CONTROL_ENDPOINT: http://consumer:9193/control - EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol - EDC_PARTICIPANT_ID: consumer - EDC_API_AUTH_KEY: password - entrypoint: java - -javaagent:/app/libs/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/resources/logging.properties - -jar /app/connector.jar - - provider: - build: - context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile - volumes: - - ./:/resources - ports: - - "8181:8181" - - "8182:8182" - environment: - EDC_HOSTNAME: provider - OTEL_SERVICE_NAME: provider - OTEL_TRACES_EXPORTER: jaeger - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 - WEB_HTTP_PORT: 8181 - WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 8182 - WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_PROTOCOL_PORT: 8282 - WEB_HTTP_PROTOCOL_PATH: /protocol - WEB_HTTP_CONTROL_PORT: 8183 - WEB_HTTP_CONTROL_PATH: /control - EDC_CONTROL_ENDPOINT: http://provider:8183/control - EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol - EDC_PARTICIPANT_ID: provider - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /resources/README.md - entrypoint: java - -javaagent:/app/libs/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/resources/logging.properties - -jar /app/connector.jar - - jaeger: - image: jaegertracing/all-in-one - ports: - - "16686:16686" - - prometheus: - image: prom/prometheus:v2.30.3 - volumes: - - ./prometheus/:/etc/prometheus/ - ports: - - "9090:9090" diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json deleted file mode 100644 index b84b7631..00000000 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "dataDestination": { - "type": "File", - "path": "/resources/README_transferred.md", - "keyName": "keyName" - }, - "protocol": "dataspace-protocol-http", - "assetId": "test-document", - "contractId": "{agreement ID}", - "connectorId": "provider", - "connectorAddress": "http://provider:8282/protocol", - "privateProperties": {} -} diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile deleted file mode 100644 index 93680a50..00000000 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:jdk17 AS build - -WORKDIR /home/gradle/project/ -COPY --chown=gradle:gradle . /home/gradle/project/ -RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-consumer:build - -FROM openjdk:17-slim - -WORKDIR /app -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile deleted file mode 100644 index 50efc2f5..00000000 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:jdk17 AS build - -WORKDIR /home/gradle/project/ -COPY --chown=gradle:gradle . /home/gradle/project/ -RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-provider:build - -FROM openjdk:17-slim - -WORKDIR /app -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-06-consumer-pull-http/README.md b/transfer/transfer-06-consumer-pull-http/README.md deleted file mode 100644 index 23ebea8c..00000000 --- a/transfer/transfer-06-consumer-pull-http/README.md +++ /dev/null @@ -1,517 +0,0 @@ -# Implement a simple "Consumer Pull" Http transfer flow - -The purpose of this example is to show a data exchange between 2 connectors, one representing the -data provider and the other, the consumer. It's based on a consumer pull usecase that you can find -more details -on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane) -For the sake of simplicity, the provider and the consumer -will be on the same machine, but in a real world configuration, they will likely be on different -machines. The final goal of this example is to present the steps through which the 2 connectors will -have to pass so that the consumer can have access to the data, held by the provider. - -Those steps are the following: - -* Running the provider connector -* Running the consumer connector -* Running a Http server that will receive the Endpoint Data Reference on the consumer side, that - contains the url to be used to get the data. -* Register data plane instance for provider connector -* Register data plane instance for consumer connector -* Create an Asset on the provider (The asset will be the data to be shared) -* Create an access policy on the provider (The policy will define the access right to the data) -* Create a contract definition on the provider - -At this step, the consumer should be able to fetch the catalog from the provider and to see the -contract offer generated from the resources that have been created. - -Once the catalog is available, to access the data, the consumer should follow the following steps: - -* Performing a contract negotiation with the provider -* Performing a transfer - * The consumer will initiate a file transfer - * The provider will send an EndpointDataReference to the consumer -* The consumer could reach the endpoint and access the data - -Also, in order to keep things organized, the code in this example has been separated into several -Java modules: - -* `connector`: contains the configuration and build files for both the - consumer and the provider connector -* `backend-service`: represent the backend service where the consumer connector will send the - EndpointDataReference to access the data - -> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single -> asset. This will be deleted after the provider shutdown. - -### Provider connector - -The provider connector is the one providing EndpointDataReference to the consumer after it initiates -a transfer. - -### Consumer connector - -The consumer is the one "requesting" the data to the provider. - -# How to build a connector - -In fact, in the configuration of our example, both the provider and the consumer are connectors. -Therefore, to set up our example, we need to start a connector with the configuration for a provider -and another one with the configuration of a consumer. - -This section allows you to build the connector before launching it. - -```bash -./gradlew transfer:transfer-06-consumer-pull-http:http-pull-connector:build -``` - -After the build end you should verify that the connector jar is created in the directory -[http-pull-connector.jar](http-pull-connector/build/libs/http-pull-connector.jar) - -# How to run a connector - -It is important to note that only the property file differs between the consumer and the provider. -You can find the configuration file in the directories below: - -* [provider](http-pull-provider/provider-configuration.properties) -* [consumer](http-pull-consumer/consumer-configuration.properties) - -The section bellow will show you some explanation about some of the properties that you can find in -the configuration files. - -#### 1. edc.receiver.http.endpoint - -This property is used to define the endpoint where the connector consumer will send the -EndpointDataReference. - -#### 2. edc.dataplane.token.validation.endpoint - -This property is used to define the endpoint exposed by the control plane to validate the token. - -## Run the connectors - -To run the provider, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-06-consumer-pull-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties -Dedc.fs.config=transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties -jar transfer/transfer-06-consumer-pull-http/http-pull-connector/build/libs/pull-connector.jar -``` - -To run a consumer, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-06-consumer-pull-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties -Dedc.fs.config=transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties -jar transfer/transfer-06-consumer-pull-http/http-pull-connector/build/libs/pull-connector.jar -``` - -Assuming you didn't change the ports in config files, the consumer will listen on the -ports `29191`, `29192` (management API) and `29292` (DSP API) and the provider will listen on the -ports `12181`, `19182` (management API) and `19282` (DSP API). - -# Run the sample - -Running this sample consists of multiple steps, that are executed one by one and following the same -order. - -> Please in case you have some issues with the jq option, not that it's not mandatory, and you can -> drop it from the command. it's just used to format the output, and the same advice should be -> applied to all calls that use `jq`. - -### 1. Register data plane instance for provider - -Before a consumer can start talking to a provider, it is necessary to register the data plane -instance of a connector. This is done by sending a POST request to the management API of the -provider connector. The request body should contain the data plane instance of the consumer -connector. - -The registration of the provider data plane instance is done by sending a POST -request to the management API of the connector. - -```bash -curl -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ "HttpData" ], - "allowedDestTypes": [ "HttpProxy", "HttpData" ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } - }' \ - -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq -``` - -### 2. Create an asset on the provider side - -The provider connector needs to transfer a file to the location specified by the consumer connector -when the data are requested. In order to offer any data, the provider must maintain an internal list -of resources offered, through a contract offer, the so-called "catalog". - -The following request creates an asset on the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "asset": { - "@id": "assetId", - "properties": { - "name": "product description", - "contenttype": "application/json" - } - }, - "dataAddress": { - "type": "HttpData", - "name": "Test asset", - "baseUrl": "https://jsonplaceholder.typicode.com/users", - "proxyPath": "true" - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ - -s | jq -``` - -> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for -> the purpose of this example. It will be the data that the consumer will pull on the sample -> execution. - -Additional properties on `HttpData` can be used to allow consumers to enrich the data request: - -- `proxyPath`: allows specifying additional path segments. -- `proxyQueryParams`: allows specifying query params. -- `proxyBody`: allows attaching a body. -- `proxyMethod`: allows specifying the Http Method (default `GET`) - -### 3. Create a policy on the provider - -In order to manage the accessibility rules of an asset, it is essential to create a policy. However, -to keep things simple, we will choose a policy that gives direct access to all the assets that are -associated within the contract definitions. -This means that the consumer connector can request any asset from the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "aPolicy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ - -s | jq -``` - -### 4. Create a contract definition on provider - -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition -associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "aPolicy", - "contractPolicyId": "aPolicy", - "assetsSelector": [] - }' -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ - -s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "1", - "edc:createdAt": 1674578184023, - ... -} -``` - -### 5. How to fetch catalog on consumer side - -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain -all the contract offers available for negotiation. In our case, it will contain a single contract -offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following -endpoint: - -```bash -curl -X POST "http://localhost:29193/management/v2/catalog/request" \ - -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "providerUrl": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" - }' -s | jq -``` - -Sample output: - -```json -{ - "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", - "@type": "dcat:Catalog", - "dcat:dataset": { - "@id": "assetId", - "@type": "dcat:Dataset", - "odrl:hasPolicy": { - "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@type": "odrl:Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - }, - "dcat:distribution": [ - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpProxy" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - }, - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpData" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - } - ], - "edc:name": "product description", - "edc:id": "assetId", - "edc:contenttype": "application/json" - }, - "dcat:service": { - "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", - "@type": "dcat:DataService", - "dct:terms": "connector", - "dct:endpointUrl": "http://localhost:19194/protocol" - }, - "edc:participantId": "anonymous", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 6. Negotiate a contract - -In order to request any data, a contract gets negotiated, and an agreement is resulting has to be -negotiated between providers and consumers. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence -looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the - provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later - reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counter offers in addition to just confirming or declining an offer. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "offer": { - "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "assetId": "assetId", - "policy": { - "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@type": "Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - } - } -}' -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ --s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", - "edc:createdAt": 1685525281848, - ... -} -``` - -### 7. Getting the contract agreement id - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. -This UUID is the ID of the ongoing contract negotiation between consumer and provider. The -negotiation sequence between provider and consumer is executed asynchronously in the background by a -state machine. Once both provider and consumer either reach the `confirmed` or the `declined` -state, the negotiation is finished. We can now use the UUID to check the current status of the -negotiation using an endpoint on the consumer side. - -```bash -curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ - --header 'Content-Type: application/json' \ - -s | jq -``` - -Sample output: - -```json -{ - "@type": "edc:ContractNegotiationDto", - "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", - "edc:type": "CONSUMER", - "edc:protocol": "dataspace-protocol-http", - "edc:state": "FINALIZED", - "edc:counterPartyAddress": "http://localhost:19194/protocol", - "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 8. Start the transfer - -As a pre-requisite, you need to have an http server that runs on port 4000 and logs all the incoming requests, it will -be mandatory to get the EndpointDataReference that will be used to get the data. - -```bash -./gradlew util:http-request-logger:build -HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar -``` - -Now that we have a contract agreement, we can finally request the file. In the request body, we need -to specify which asset we want transferred, the ID of the contract agreement, the address of the -provider connector and where we want the file transferred. You will find the request body below. -Before executing the request, insert the contract agreement ID from the previous step. Then run : - -> the "HttpProxy" method is used for the consumer pull method, and it means that it will be up to -> the consumer to request the data to the provider and that the request will be a proxy for the -> datasource - -```bash -curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ - -H "Content-Type: application/json" \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "contractId": "", - "assetId": "assetId", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpProxy" - } - }' \ - -s | jq -``` - -Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( -process id) created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and -performed asynchronously. - -Sample output: - -```json -{ - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:createdAt": 1674078357807, - ... -} -``` - -### 9. Check the transfer status - -Due to the nature of the transfer, it will be very fast and most likely already done by the time you -read the UUID. - -```bash -curl http://localhost:29193/management/v2/transferprocesses/ -``` - - -You should see the Transfer Process in `COMPLETED` state: -```json -{ - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:state": "COMPLETED", - ... -} - -``` - -### 10. Pull the data - -At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed -to get the data from the provider: - -```json -{ - "id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "endpoint": "http://localhost:29291/public/", - "authKey": "Authorization", - "authCode": "eyJhbGciOiJSUzI1NiJ9.eyJkYWQiOiJ7XCJwcm9wZXJ0aWVzXCI6e1wiYXV0aEtleVwiOlwiQXV0aG9yaXphdGlvblwiLFwiYmFzZVVybFwiOlwiaHR0cDpcL1wvbG9jYWxob3N0OjE5MjkxXC9wdWJsaWNcL1wiLFwiYXV0aENvZGVcIjpcImV5SmhiR2NpT2lKU1V6STFOaUo5LmV5SmtZV1FpT2lKN1hDSndjbTl3WlhKMGFXVnpYQ0k2ZTF3aVltRnpaVlZ5YkZ3aU9sd2lhSFIwY0hNNlhDOWNMMnB6YjI1d2JHRmpaV2h2YkdSbGNpNTBlWEJwWTI5a1pTNWpiMjFjTDNWelpYSnpYQ0lzWENKdVlXMWxYQ0k2WENKVVpYTjBJR0Z6YzJWMFhDSXNYQ0owZVhCbFhDSTZYQ0pJZEhSd1JHRjBZVndpZlgwaUxDSmxlSEFpT2pFMk56UTFPRGcwTWprc0ltTnBaQ0k2SWpFNk1XVTBOemc1TldZdE9UQXlOUzAwT1dVeExUazNNV1F0WldJNE5qVmpNemhrTlRRd0luMC5ITFJ6SFBkT2IxTVdWeWdYZi15a0NEMHZkU3NwUXlMclFOelFZckw5eU1tQjBzQThwMHFGYWV0ZjBYZHNHMG1HOFFNNUl5NlFtNVU3QnJFOUwxSE5UMktoaHFJZ1U2d3JuMVhGVUhtOERyb2dSemxuUkRlTU9ZMXowcDB6T2MwNGNDeFJWOEZoemo4UnVRVXVFODYwUzhqbU4wZk5sZHZWNlFpUVFYdy00QmRTQjNGYWJ1TmFUcFh6bDU1QV9SR2hNUGphS2w3RGsycXpJZ0ozMkhIdGIyQzhhZGJCY1pmRk12aEM2anZ2U1FieTRlZXU0OU1hclEydElJVmFRS1B4ajhYVnI3ZFFkYV95MUE4anNpekNjeWxyU3ljRklYRUV3eHh6Rm5XWmczV2htSUxPUFJmTzhna2RtemlnaXRlRjVEcmhnNjZJZzJPR0Eza2dBTUxtc3dcIixcInByb3h5TWV0aG9kXCI6XCJ0cnVlXCIsXCJwcm94eVF1ZXJ5UGFyYW1zXCI6XCJ0cnVlXCIsXCJwcm94eUJvZHlcIjpcInRydWVcIixcInR5cGVcIjpcIkh0dHBEYXRhXCIsXCJwcm94eVBhdGhcIjpcInRydWVcIn19IiwiZXhwIjoxNjc0NTg4NDI5LCJjaWQiOiIxOjFlNDc4OTVmLTkwMjUtNDllMS05NzFkLWViODY1YzM4ZDU0MCJ9.WhbTzERmM75mNMUG2Sh-8ZW6uDQCus_5uJPvGjAX16Ucc-2rDcOhAxrHjR_AAV4zWjKBHxQhYk2o9jD-9OiYb8Urv8vN4WtYFhxJ09A0V2c6lB1ouuPyCA_qKqJEWryTbturht4vf7W72P37ERo_HwlObOuJMq9CS4swA0GBqWupZHAnF-uPIQckaS9vLybJ-gqEhGxSnY4QAZ9-iwSUhkrH8zY2GCDkzAWIPmvtvRhAs9NqVkoUswG-ez1SUw5bKF0hn2OXv_KhfR8VsKKYUbKDQf5Wagk7rumlYbXMPNAEEagI4R0xiwKWVTfwwZPy_pYnHE7b4GQECz3NjhgdIw", - "properties": { - "cid": "1:1e47895f-9025-49e1-971d-eb865c38d540" - } -} -``` - -Once this json is read, use a tool like postman or curl to execute the following query, to read the -data - -```bash -curl --location --request GET 'http://localhost:29291/public/' --header 'Authorization: ' -``` - -At the end, and to be sure that you correctly achieved the pull, you can check if the data you get -is the same as the one you can get at https://jsonplaceholder.typicode.com/users - - -Since we configured the `HttpData` with `proxyPath`, we could also ask for a specific user with: - -```bash -curl --location --request GET 'http://localhost:29291/public/1' --header 'Authorization: ' -``` - -And the data returned will be the same as in https://jsonplaceholder.typicode.com/users/1 diff --git a/transfer/transfer-07-provider-push-http/README.md b/transfer/transfer-07-provider-push-http/README.md deleted file mode 100644 index a6769761..00000000 --- a/transfer/transfer-07-provider-push-http/README.md +++ /dev/null @@ -1,445 +0,0 @@ -# Implement a simple "Provider Push" Http transfer flow - -The purpose of this example is to show a data exchange between two connectors, one representing the -data provider and the other, the consumer. It's based on a provider push use case that you can find -more details -on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). -For the sake of simplicity, the provider and the consumer -will be on the same machine, but in a real world configuration, they will likely be on different -machines. The final goal of this example is to present the steps through which the two connectors will -have to pass so that the consumer can have access to the data, held by the provider. - -Those steps are the following: - -* Running the provider connector -* Running the consumer connector -* Register data plane instance for provider connector -* Create an Asset on the provider (The asset will be the data to be shared) -* Create an access policy on the provider (The policy will define the access right to the data) -* Create a contract definition on the provider - -At this step, the consumer should be able to fetch the catalog from the provider and to see the -contract offer generated from the resources that have been created. - -Once the catalog is available, to access the data, the sample will go through: - -* Performing a contract negotiation with the provider -* Performing a transfer - * The consumer will initiate a file transfer - * Provider Control Plane retrieves the DataAddress of the actual data source and creates a - DataFlowRequest based on the received DataRequest and this data address -* Provider Data Plane fetches data from the actual data source -* Provider Data Plane pushes data to the consumer service - -Also, in order to keep things organized, the code in this example has been separated into one -Java module: - -* `connector`: contains the configuration and build files for both the - consumer and the provider connector -* `provider-push-backend-service`: represent the backend service where the provider will push the data after the - consumer has initiated the transfer - -> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single -> asset. This will be deleted after the provider shutdown. - -# How to build a connector - -In fact, in the configuration of our example, both the provider and the consumer are connectors. -Therefore, to set up our example, we need to start a connector with the configuration for a provider -and another one with the configuration of a consumer. - -This section allows you to build the connector before launching it. - -```bash -./gradlew transfer:transfer-07-provider-push-http:http-push-connector:build -``` - -After the building end, you should verify that the connector jar is created in the directory -[http-push-connector.jar](http-push-connector/build/libs/http-push-connector.jar) - -# How to run a connector - -It is important to note that only the property file differs between the consumer and the provider. -You can find the configuration file in the directories below: - -* [provider](http-push-provider/provider-configuration.properties) -* [consumer](http-push-consumer/consumer-configuration.properties) - -The section bellow will show you some explanation about some of the properties that you can find in -the configuration files. - -## Run the connectors - -To run the provider, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-07-provider-push-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-07-provider-push-http/http-push-provider/provider-vault.properties -Dedc.fs.config=transfer/transfer-07-provider-push-http/http-push-provider/provider-configuration.properties -jar transfer/transfer-07-provider-push-http/http-push-connector/build/libs/push-connector.jar -``` - -To run the consumer, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-07-provider-push-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-07-provider-push-http/http-push-consumer/consumer-vault.properties -Dedc.fs.config=transfer/transfer-07-provider-push-http/http-push-consumer/consumer-configuration.properties -jar transfer/transfer-07-provider-push-http/http-push-connector/build/libs/push-connector.jar -``` - -Assuming you didn't change the ports in config files, the consumer will listen on the -ports `29191`, `29193` (management API) and `29194` (DSP API) and the provider will listen on the -ports `19191`, `19193` (management API) and `19194` (DSP API). - -# Run the sample - -Running this sample consists of multiple steps, that are executed one by one and following the same -order. - -> Please in case you have some issues with the jq option, not that it's not mandatory, and you can -> drop it from the command. it's just used to format the output, and the same advice should be -> applied to all calls that use `jq`. - -### 1. Register data plane instance for provider - -Before a consumer can start talking to a provider, it is necessary to register the data plane -instance of a connector. This is done by sending a POST request to the management API of the -provider connector. The request body should contain the data plane instance of the consumer -connector. - -The registration of the provider data plane instance is done by sending a POST -request to the management API of the connector. - -```bash -curl -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ "HttpData" ], - "allowedDestTypes": [ "HttpProxy", "HttpData" ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } - }' \ - -X POST "http://localhost:19193/management/v2/dataplanes" -``` - -### 2. Create an Asset on the provider side - -The provider connector needs to transfer a file to the location specified by the consumer connector -when the data are requested. In order to offer any data, the provider must maintain an internal list -of resources offered, through a contract offer, the so-called "catalog". - -The following request creates an asset on the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "asset": { - "@id": "assetId", - "properties": { - "name": "product description", - "contenttype": "application/json" - } - }, - "dataAddress": { - "name": "Test asset", - "baseUrl": "https://jsonplaceholder.typicode.com/users", - "type": "HttpData" - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ - -s | jq -``` - -> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for -> the purpose of this example. It will be the data that the provider will push on the sample -> execution. - -### 3. Create a Policy on the provider - -In order to manage the accessibility rules of an asset, it is essential to create a policy. However, -to keep things simple, we will choose a policy that gives direct access to all the assets that are -associated within the contract definitions. -This means that the consumer connector can request any asset from the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "aPolicy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ - -s | jq -``` - -### 4. Create a contract definition on Provider - -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition -associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "aPolicy", - "contractPolicyId": "aPolicy", - "assetsSelector": [] - }' -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ - -s | jq -``` - -Sample output: - -```json -{ - "createdAt": 1674578184023, - "id": "1" -} -``` - -### 5. How to fetch catalog on consumer side - -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain -all the contract offers available for negotiation. In our case, it will contain a single contract -offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following -endpoint: - -```bash -curl -X POST "http://localhost:29193/management/v2/catalog/request" \ - -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "providerUrl": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" - }' -s | jq -``` - -Sample output: - -```json -{ - "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", - "@type": "dcat:Catalog", - "dcat:dataset": { - "@id": "assetId", - "@type": "dcat:Dataset", - "odrl:hasPolicy": { - "@id": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "@type": "odrl:Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - }, - "dcat:distribution": [ - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpProxy" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - }, - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpData" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - } - ], - "edc:name": "product description", - "edc:id": "assetId", - "edc:contenttype": "application/json" - }, - "dcat:service": { - "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", - "@type": "dcat:DataService", - "dct:terms": "connector", - "dct:endpointUrl": "http://localhost:19194/protocol" - }, - "edc:participantId": "anonymous", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 6. Negotiate a contract - -In order to request any data, a contract gets negotiated, and an agreement is resulting has to be -negotiated between providers and consumers. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence -looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the - provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later - reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counteroffers in addition to just confirming or declining an offer. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "offer": { - "offerId": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "assetId": "assetId", - "policy": { - "@id": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "@type": "Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - } - } -}' -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ --s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", - "edc:createdAt": 1685525281848, - ... -} -``` - -### 7. Getting the contract agreement id - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. -This UUID is the ID of the ongoing contract negotiation between consumer and provider. The -negotiation sequence between provider and consumer is executed asynchronously in the background by a -state machine. Once both provider and consumer either reach the `confirmed` or the `declined` -state, the negotiation is finished. We can now use the UUID to check the current status of the -negotiation using an endpoint on the consumer side. - -```bash -curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ - --header 'Content-Type: application/json' \ - -s | jq -``` - -Sample output: - -```json -{ - "@type": "edc:ContractNegotiationDto", - "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", - "edc:type": "CONSUMER", - "edc:protocol": "dataspace-protocol-http", - "edc:state": "FINALIZED", - "edc:counterPartyAddress": "http://localhost:19194/protocol", - "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:MjQ2ODMxOTMtZmFhMS00MzMxLWE2YzYtYTQ1ZjNkNzJkYWNk", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 8. Start the transfer - -As a pre-requisite, you need to have an http server that runs on port 4000 and logs all the incoming requests, it will -be mandatory to get the data from the provider. - -```bash -./gradlew util:http-request-logger:build -HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar -``` - -Now that we have a contract agreement, we can finally request the file. In the request body, we need -to specify which asset we want transferred, the ID of the contract agreement, the address of the -provider connector and where we want the file transferred. You will find the request body below. -Before executing the request, insert the contract agreement ID from the previous step. Then run : - -> keep in mind that, to make a transfer with a provider push method, the dataDestination type should -> be any value different from the "HttpProxy". - -```bash -curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ - -H "Content-Type: application/json" \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "contractId": "", - "assetId": "assetId", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpData", - "baseUrl": "http://localhost:4000/api/consumer/store" - } - }' \ - -s | jq -``` - -Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( -process id) created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and -performed asynchronously. - -Sample output: - -```json - { - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:createdAt": 1674078357807, - ... -} -``` - -### 9. Check the transfer status - -Due to the nature of the transfer, it will be very fast and most likely already done by the time you -read the UUID. - -```bash -curl http://localhost:29193/management/v2/transferprocesses/ -``` - -### 10. Check the data - -At this step, you can check the data by checking the log of the http server exposed on port 4000, you should see a log -that shows the same data that you can get from https://jsonplaceholder.typicode.com/users diff --git a/transfer/transfer-07-provider-push-http/certs/cert.pem b/transfer/transfer-07-provider-push-http/certs/cert.pem deleted file mode 100644 index c7dc26fa..00000000 --- a/transfer/transfer-07-provider-push-http/certs/cert.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy -MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g -E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS -PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H -I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W -EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 -h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud -DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B -2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn -QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ -rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe -Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy -+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR -IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ -g3h+15GuzbsSzOCOEYOT ------END CERTIFICATE----- diff --git a/transfer/transfer-07-provider-push-http/certs/cert.pfx b/transfer/transfer-07-provider-push-http/certs/cert.pfx deleted file mode 100644 index 7ac9c73e0eb6b5c2fa625d818e42d55bdafc8a2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2477 zcmY+^c{J3G8V7K*W@N}R7=$7FHnNl@TSGL2Fj;DNOC*GhE&IP}zqvy)xzv zQL;p;A(BR~v1ZAx(5-Xsz3;t$Jm)#*`#tCR@A+T>L=6}O!2$?>I6^7OB54~5Is~Es z2zMxexcDn}!U9m>;!u#NoRD=ju#8QFzm^2$y49g1TMv z=T@sTo@Iuph_A%yU=lxc7M{sJMHUT>F|}KXUgo$hFFS>A2!j7*6*i&bNm83Pw2;<1 zlD}V5KUbh&o|jOxsP+BiwXBG4&`tucGj8`-Hm@xYJ}`H^&q$4GmUu<1YCBvj`C$0i zkFsaL3jy!*E?2AUSW6Pe-Zjg?APSh+Uhq-Q)U@7jmt#8E>T$EH6!|pKz~;1w=$~nhB-?cONKvYWB(}r)QbC0-GdWIqew0w+G5Wq#m^rPtZ{$ z&t3^ej`3$~G<<2pp49+5+q{j;Rh)KZXRnVO3qF5>eHL=xcsdXrkEpgxwm0V8`ssk6Z8vc4Et<{a&YT958$yF6j}K( z4hN^-U5JL_{5>%{Y?3HMe0}$$xMM|0Q~;4aZqhSedSM+=_Eg(Xx&_xr&h4{kQ6=Zy zW@OQ7;(7v+Fweo zb}gS9>{w6uxhvyvlziQ9z-3d+n125oDs)1&%G zEwA3(y*}qIWtKDR8WA@gI9QkkDNyw1yM+2Q51Ffz7>M>4w=m^={|4pZ%S)yOfKw=S zz69`I8q8wN^N`NiDcpnDll_@y_Lt-uFCKP{LaIrnK#~AY;j~X%jH{UC%6?YFF(6`J zeO0vTRy6aht=iu+Gn1WZZ9${yya0`i>EAky_j=kWwxgJ@qBH-e;dkS@+_oyS3!Aiw z;flVSZ2Gwg5uLIPua4c7LXmk++;IQ)ShM>oI!v-Ny??463&2eN=kX~3>^KyFsr`zT zevKUb@P8QK0)c;dvHXjWh5yq5@>_>7Tkb5#{9e;<9iRa8RoC_4ISv`^=M(qQdQ#2V zIza+e3IWHj4>i1V?+l92n8{c(>>O7{qXrb~$} z8V6i+B42m1p07qSh%@E`sm?9n2@PvNYy1jX*e|%F=6kUedQy;Iln(yLlX1`aN#?b- zZwXNhKWgoMN~ESsM{=(WNU3B-P_1gdul?TFtyYb>%F_uP1wj4yp>Z!i6*k*Ul%Gz)g&^5L%mg;3Yei{>pW3}~9}Fie!CscOnc zJgrKau67c@sA=-R{3>m^F}gVEM{D)BD;Rfv&5G3Fy=WURVqP6QJxj82TTmc;y4Q1- zx|64G<-u>)G8wNcEv5P}fF>N`2APM^GsZsD9(GGFY8a1t-ibjD#{`UD_>(v-XlYzo zA;v?EMrJqVD#h=r6cyea-Ed8TH>Zq;HKlDnF#HM0hMbMSu#w}iv=RZVfG{~?FaJbD zMa^hLzlX||5&8d(&Eyl0P5RU*~LQSx?Wln6}c1 zjE%SIZYd-En@6!L4H>W9OOz7Uh%swV~-_xx9L37=5~uCx-Us8{Wfv z!)kuI6u*;E^(MO0p!ubmp{22ExfFD%2=4oC9Gi#0bMN>{7MOeNuCxxli^wSO z#`0sH5ncdKZJqBDq$!_D$m6M1M9n|sZbu%mtl$5t$fH@XWMBH^*&PYhQKlKIzN#AS zP+p19Vo>>tV1Zg+=2Mjn;E1Epntc)Ceohe56KASXPy5a+R<=Vut-5#zROAXC2-6Qq!-V_HxY5dt)>gm+(P2v_EXSK~h`}Bm!(8j7N zDp}fG?p^A7UZ|tg4P;$YSJ-!8MYi_%t?<-l^SUW^yH23Ug-IiYTF2H#qp7+WS4$R* zaAti?s5msEyR0#3X@J~V5>*x`)cC0CVNqh9tGDxxLcyCjv}KFV2j%^Du{Tt5oE7^+ z;-xx^OdO|Xz6kXWC+|Nr=h33y@q4B#HOS`(w#wQyP1IL;yd7dDd?TSkv`?1vgrI`tLtzxEjlPjW_Z(lo%ao_L5vmNUfyH1EaF{X=1S|^U nfC>)L0b!m6KEg{ { manifest { attributes["Main-Class"] = "org.eclipse.edc.samples.util.HttpRequestLoggerServer" } + archiveFileName.set("http-request-logger.jar") }