From e2387fc2e5560c1b84c5a3fd33a776d6792aa0e7 Mon Sep 17 00:00:00 2001 From: ndr_brt Date: Mon, 10 Jun 2024 11:01:25 +0200 Subject: [PATCH] build(deps): Bump EDC to 0.7.0 (#268) * build(deps): Bump EDC to 0.7.0 * cleanup * Fix pull sample * avoid data plane registration * fix open-telemetry sample --- advanced/advanced-01-open-telemetry/README.md | 63 +++++++------- .../docker-compose.yaml | 23 +++-- .../open-telemetry-runtime/Dockerfile | 12 +-- .../open-telemetry-runtime/build.gradle.kts | 8 +- .../sample/runtime/SeedVaultExtension.java | 87 +++++++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../resources/negotiate-contract.json | 3 - gradle/libs.versions.toml | 16 ++-- system-tests/build.gradle.kts | 4 + .../advanced/Advanced01openTelemetryTest.java | 5 +- .../samples/common/PrerequisitesCommon.java | 40 +-------- .../transfer/Transfer00prerequisitesTest.java | 39 --------- .../transfer/Transfer01negotiationTest.java | 2 - .../transfer/Transfer02consumerPullTest.java | 38 ++++---- .../transfer/Transfer03providerPushTest.java | 2 - .../transfer/Transfer04eventConsumerTest.java | 2 - .../streaming/Streaming01httpToHttpTest.java | 2 - .../streaming/Streaming02KafkaToHttpTest.java | 4 - .../streaming/StreamingParticipant.java | 28 ------ .../edc/samples/util/TransferUtil.java | 9 +- .../streaming-01-http-to-http/README.md | 9 -- .../streaming-01-http-to-http/dataplane.json | 9 -- .../streaming-01-runtime/build.gradle.kts | 3 +- .../streaming-02-kafka-to-http/README.md | 10 --- .../streaming-02-runtime/build.gradle.kts | 3 +- .../streaming-03-runtime/build.gradle.kts | 2 +- transfer/transfer-00-prerequisites/README.md | 37 +------- .../connector/build.gradle.kts | 9 +- .../sample/runtime/SeedVaultExtension.java | 87 +++++++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../resources/certs/cert.pem | 21 ----- .../resources/certs/key.pem | 28 ------ .../consumer-configuration.properties | 6 +- .../configuration/consumer-vault.properties | 1 - .../provider-configuration.properties | 7 +- .../configuration/provider-vault.properties | 1 - .../register-data-plane-consumer.json | 17 ---- .../register-data-plane-provider.json | 17 ---- transfer/transfer-02-consumer-pull/README.md | 47 +++++----- transfer/transfer-03-provider-push/README.md | 16 +++- transfer/transfer-04-event-consumer/README.md | 2 +- .../consumer-with-listener/build.gradle.kts | 3 +- .../cloud-transfer-consumer/build.gradle.kts | 2 +- .../cloud-transfer-consumer/config.properties | 3 +- .../cloud-transfer-provider/build.gradle.kts | 2 +- .../cloud-transfer-provider/config.properties | 3 +- .../transfer-file-cloud/build.gradle.kts | 2 +- 47 files changed, 325 insertions(+), 411 deletions(-) create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java delete mode 100644 transfer/streaming/streaming-01-http-to-http/dataplane.json create mode 100644 transfer/transfer-00-prerequisites/connector/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java create mode 100644 transfer/transfer-00-prerequisites/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-00-prerequisites/resources/certs/cert.pem delete mode 100644 transfer/transfer-00-prerequisites/resources/certs/key.pem delete mode 100644 transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties delete mode 100644 transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties delete mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json delete mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json diff --git a/advanced/advanced-01-open-telemetry/README.md b/advanced/advanced-01-open-telemetry/README.md index 5186c4ef..54bb5e40 100644 --- a/advanced/advanced-01-open-telemetry/README.md +++ b/advanced/advanced-01-open-telemetry/README.md @@ -25,29 +25,17 @@ is configured to expose a Prometheus metrics endpoint. To run the consumer, the provider, and Jaeger execute the following commands in the project root folder: +Build the connector ```bash -docker compose -f advanced/advanced-01-open-telemetry/docker-compose.yaml up --abort-on-container-exit +./gradlew :advanced:advanced-01-open-telemetry:open-telemetry-runtime:build ``` -Open a new terminal. - -Register data planes for provider and consumer: - +Start the docker compose ```bash -curl -H 'Content-Type: application/json' \ - -H "X-Api-Key: password" \ - -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ - -X POST "http://localhost:19193/management/v2/dataplanes" \ - -s | jq +docker compose -f advanced/advanced-01-open-telemetry/docker-compose.yaml up --abort-on-container-exit ``` -```bash -curl -H 'Content-Type: application/json' \ - -H "X-Api-Key: password" \ - -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json \ - -X POST "http://localhost:29193/management/v2/dataplanes" \ - -s | jq -``` +Open a new terminal. Create an asset: @@ -95,40 +83,55 @@ The output will be something like: "@id": "assetId", "@type": "dcat:Dataset", "odrl:hasPolicy": { - "@id": "MQ==:YXNzZXRJZA==:YjI5ZDVkZDUtZWU0Mi00NWRiLWE2OTktYjNmMjlmMWNjODk3", - "@type": "odrl:Set", + "@id": "MQ==:YXNzZXRJZA==:NjdlNDFhM2EtYThjMS00YTBmLWFkNmYtMjk5NzkzNTE2OTE3", + "@type": "odrl:Offer", "odrl:permission": [], "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" + "odrl:obligation": [] }, "dcat:distribution": [ { "@type": "dcat:Distribution", "dct:format": { - "@id": "HttpProxy" + "@id": "HttpData-PULL" }, - "dcat:accessService": "06348bca-6bf0-47fe-8bb5-6741cff7a955" + "dcat:accessService": { + "@id": "cb701b36-48ee-4132-8436-dba7b83c606c", + "@type": "dcat:DataService", + "dcat:endpointDescription": "dspace:connector", + "dcat:endpointUrl": "http://provider:19194/protocol", + "dct:terms": "dspace:connector", + "dct:endpointUrl": "http://provider:19194/protocol" + } }, { "@type": "dcat:Distribution", "dct:format": { - "@id": "HttpData" + "@id": "HttpData-PUSH" }, - "dcat:accessService": "06348bca-6bf0-47fe-8bb5-6741cff7a955" + "dcat:accessService": { + "@id": "cb701b36-48ee-4132-8436-dba7b83c606c", + "@type": "dcat:DataService", + "dcat:endpointDescription": "dspace:connector", + "dcat:endpointUrl": "http://provider:19194/protocol", + "dct:terms": "dspace:connector", + "dct:endpointUrl": "http://provider:19194/protocol" + } } ], - "edc:name": "product description", - "edc:id": "assetId", - "edc:contenttype": "application/json", + "name": "product description", + "id": "assetId", + "contenttype": "application/json", "@context": { - "dct": "https://purl.org/dc/terms/", + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", + "dcat": "http://www.w3.org/ns/dcat#", + "dct": "http://purl.org/dc/terms/", "odrl": "http://www.w3.org/ns/odrl/2/", "dspace": "https://w3id.org/dspace/v0.8/" } } + ``` With the `odrl:hasPolicy/@id` we can now replace it in the [negotiate-contract.json](resources/negotiate-contract.json) file diff --git a/advanced/advanced-01-open-telemetry/docker-compose.yaml b/advanced/advanced-01-open-telemetry/docker-compose.yaml index ef1d1aed..f918fe2b 100644 --- a/advanced/advanced-01-open-telemetry/docker-compose.yaml +++ b/advanced/advanced-01-open-telemetry/docker-compose.yaml @@ -4,8 +4,7 @@ services: consumer: build: - context: ../.. - dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile + context: open-telemetry-runtime volumes: - ./:/open-telemetry - ../../transfer/transfer-00-prerequisites/:/prerequisites @@ -14,11 +13,10 @@ services: - "29194:29194" environment: EDC_HOSTNAME: consumer - OTEL_SERVICE_NAME: consumer - OTEL_JAVAAGENT_EXTENSIONS: /app/opentelemetry-exporter-jaeger.jar - OTEL_TRACES_EXPORTER: jaeger + OTEL_SERVICE_NAME: provider OTEL_LOGS_EXPORTER: none - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_JAVAAGENT_EXTENSIONS: /app/opentelemetry-exporter-otlp.jar + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 OTEL_METRICS_EXPORTER: prometheus WEB_HTTP_PORT: 29191 WEB_HTTP_PATH: /api @@ -36,7 +34,6 @@ services: EDC_API_AUTH_KEY: password EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx EDC_KEYSTORE_PASSWORD: 123456 - EDC_VAULT: /prerequisites/resources/configuration/provider-vault.properties EDC_FS_CONFIG: /prerequisites/resources/configuration/provider-configuration.properties entrypoint: java -javaagent:/app/opentelemetry-javaagent.jar @@ -45,8 +42,7 @@ services: provider: build: - context: ../.. - dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile + context: open-telemetry-runtime volumes: - ./:/open-telemetry - ../../transfer/transfer-00-prerequisites/:/prerequisites @@ -57,9 +53,8 @@ services: EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider OTEL_LOGS_EXPORTER: none - OTEL_JAVAAGENT_EXTENSIONS: /app/opentelemetry-exporter-jaeger.jar - OTEL_TRACES_EXPORTER: jaeger - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_JAVAAGENT_EXTENSIONS: /app/opentelemetry-exporter-otlp.jar + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 WEB_HTTP_PORT: 19191 WEB_HTTP_PATH: /api WEB_HTTP_PUBLIC_PORT: 19291 @@ -76,7 +71,6 @@ services: EDC_API_AUTH_KEY: password EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx EDC_KEYSTORE_PASSWORD: 123456 - EDC_VAULT: /prerequisites/resources/configuration/consumer-vault.properties EDC_FS_CONFIG: /prerequisites/resources/configuration/consumer-configuration.properties EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /open-telemetry/README.md entrypoint: java @@ -86,7 +80,10 @@ services: jaeger: image: jaegertracing/all-in-one + environment: + - COLLECTOR_OTLP_ENABLED=true ports: + - "4317:4317" - "16686:16686" prometheus: diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile index 93b17717..1b65b93b 100644 --- a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile +++ b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/Dockerfile @@ -1,12 +1,6 @@ -FROM gradle:jdk17 AS build - -WORKDIR /home/gradle/project/ -COPY --chown=gradle:gradle . /home/gradle/project/ -RUN gradle advanced:advanced-01-open-telemetry:open-telemetry-runtime:build - FROM openjdk:17-slim WORKDIR /app -COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build/libs/opentelemetry-javaagent-*.jar /app/opentelemetry-javaagent.jar -COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build/libs/opentelemetry-exporter-jaeger-*.jar /app/opentelemetry-exporter-jaeger.jar -COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build/libs/connector.jar /app/connector.jar +COPY build/libs/opentelemetry-javaagent-2.4.0.jar /app/opentelemetry-javaagent.jar +COPY build/libs/opentelemetry-exporter-otlp-1.39.0.jar /app/opentelemetry-exporter-otlp.jar +COPY build/libs/connector.jar /app/connector.jar diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build.gradle.kts b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build.gradle.kts index 49089b04..b6c72176 100644 --- a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build.gradle.kts +++ b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/build.gradle.kts @@ -31,16 +31,16 @@ dependencies { implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.vault.filesystem) implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) - implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.data.plane.selector.api) implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.self.registration) implementation(libs.edc.data.plane.control.api) implementation(libs.edc.data.plane.public.api) implementation(libs.edc.data.plane.core) @@ -49,7 +49,7 @@ dependencies { implementation(libs.edc.api.observability) implementation(libs.edc.auth.tokenbased) - implementation(libs.opentelemetry.exporter.jaeger) + implementation(libs.opentelemetry.exporter.otlp) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -68,7 +68,7 @@ tasks.register("copyOpenTelemetryJar", Copy::class) { dependencies { openTelemetry(libs.opentelemetry.javaagent) - openTelemetry(libs.opentelemetry.exporter.jaeger) + openTelemetry(libs.opentelemetry.exporter.otlp) } from(openTelemetry) diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java new file mode 100644 index 00000000..171d96ff --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.sample.runtime; + +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +public class SeedVaultExtension implements ServiceExtension { + + @Inject + private Vault vault; + + private static final String PUBLIC_KEY = """ + -----BEGIN CERTIFICATE----- + MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL + BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM + GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy + MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw + HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB + AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g + E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS + PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H + I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W + EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 + h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud + DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B + 2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn + QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ + rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe + Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy + +NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR + IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ + g3h+15GuzbsSzOCOEYOT + -----END CERTIFICATE----- + """; + + private static final String PRIVATE_KEY = """ + -----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBl6XaJnXTL+6D + Wip3aBhU+MzmY4d1V9hbTm1tiZ3gE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7 + EddidN0ITHB9cQNdAfdUJ5njmsGSPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7 + DHacZT/+OztBH1RwkG2ymM94Hf8HI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvj + X5qASakBtXISKIsOU84N0/2HDN3WEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga7 + 75bPXN3M+JTSaIKE7dZbKzvx0Zi0h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2 + YmnneYoVAgMBAAECggEBAJHXiN6bctAyn+DcoHlsNkhtVw+Jk5bXIutGXjHTJtiU + K//siAGC78IZMyXmi0KndPVCdBwShROVW8xWWIiXuZxy2Zvm872xqX4Ah3JsN7/Q + NrXdVBUDo38zwIGkxqIfIz9crZ4An+J/eq5zaTfRHzCLtswMqjRS2hFeBY5cKrBY + 4bkSDGTP/c5cP7xS/UwaiTR2Ptd41f4zTyd4l5rl30TYHpazQNlbdxcOV4jh2Rnp + E0+cFEvEfeagVq7RmfBScKG5pk4qcRG0q2QHMyK5y00hdYvhdRjSgN7xIDkeO5B8 + s8/tSLU78nCl2gA9IKxTXYLitpISwZ81Q04mEAKRRtECgYEA+6lKnhn//aXerkLo + ZOLOjWQZhh005jHdNxX7DZqLpTrrfxc8v15KWUkAK1H0QHqYvfPrbbsBV1MY1xXt + sKmkeu/k8fJQzCIvFN4K2J5W5kMfq9PSw5d3XPeDaQuXUVaxBVp0gzPEPHmkKRbA + AkUqY0oJwA9gMKf8dK+flmLZfbsCgYEAxO4Roj2G46/Oox1GEZGxdLpiMpr9rEdR + JlSZ9kMGfddNLV7sFp6yPXDcyc/AOqeNj7tw1MyoT3Ar454+V0q83EZzCXvs4U6f + jUrfFcoVWIwf9AV/J4KWzMIzfqPIeNwqymZKd6BrZgcXXvAEPWt27mwO4a1GhC4G + oZv0t3lAsm8CgYAQ8C0IhSF4tgBN5Ez19VoHpDQflbmowLRt77nNCZjajyOokyzQ + iI0ig0pSoBp7eITtTAyNfyew8/PZDi3IVTKv35OeQTv08VwP4H4EZGve5aetDf3C + kmBDTpl2qYQOwnH5tUPgTMypcVp+NXzI6lTXB/WuCprjy3qvc96e5ZpT3wKBgQC8 + Xny/k9rTL/eYTwgXBiWYYjBL97VudUlKQOKEjNhIxwkrvQBXIrWbz7lh0Tcu49al + BcaHxru4QLO6pkM7fGHq0fh3ufJ8EZjMrjF1xjdk26Q05o0aXe+hLKHVIRVBhlfo + ArB4fRo+HcpdJXjox0KcDQCvHe+1v9DYBTWvymv4QQKBgBy3YH7hKz35DcXvA2r4 + Kis9a4ycuZqTXockO4rkcIwC6CJp9JbHDIRzig8HYOaRqmZ4a+coqLmddXr2uOF1 + 7+iAxxG1KzdT6uFNd+e/j2cdUjnqcSmz49PRtdDswgyYhoDT+W4yVGNQ4VuKg6a3 + Z3pC+KTdoHSKeA2FyAGnSUpD + -----END PRIVATE KEY----- + """; + + @Override + public void initialize(ServiceExtensionContext context) { + vault.storeSecret("public-key", PUBLIC_KEY); + vault.storeSecret("private-key", PRIVATE_KEY); + } +} diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..eaca6845 --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.runtime.SeedVaultExtension diff --git a/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json index 932f5a8b..44b73204 100644 --- a/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json +++ b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json @@ -9,9 +9,6 @@ "@context": "http://www.w3.org/ns/odrl.jsonld", "@id": "{{contract-offer-id}}", "@type": "Offer", - "permission": [], - "prohibition": [], - "obligation": [], "assigner": "provider", "target": "assetId" } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dbf89dc1..28599df1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ format.version = "1.1" [versions] assertj = "3.26.0" awaitility = "4.2.1" -edc = "0.6.4" +edc = "0.7.0" jakarta-json = "2.0.1" junit-pioneer = "2.2.0" jupiter = "5.10.2" @@ -29,7 +29,7 @@ edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version. edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } edc-control-plane-spi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } edc-data-plane-control-api = { module = "org.eclipse.edc:data-plane-control-api", version.ref = "edc" } -edc-data-plane-public-api = { module = "org.eclipse.edc:data-plane-public-api", version.ref = "edc" } +edc-data-plane-public-api = { module = "org.eclipse.edc:data-plane-public-api-v2", version.ref = "edc" } edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } edc-data-plane-azure-storage = { module = "org.eclipse.edc:data-plane-azure-storage", version.ref = "edc" } edc-data-plane-client = { module = "org.eclipse.edc:data-plane-client", version.ref = "edc" } @@ -38,9 +38,13 @@ edc-data-plane-http = { module = "org.eclipse.edc:data-plane-http", version.ref edc-data-plane-kafka = { module = "org.eclipse.edc:data-plane-kafka", version.ref = "edc" } edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" } edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } +edc-data-plane-self-registration = { module = "org.eclipse.edc:data-plane-self-registration", version.ref = "edc" } edc-data-plane-spi = { module = "org.eclipse.edc:data-plane-spi", version.ref = "edc" } edc-data-plane-util = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" } edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } +edc-edr-cache-api = { module = "org.eclipse.edc:edr-cache-api", version.ref = "edc" } +edc-edr-store-core = { module = "org.eclipse.edc:edr-store-core", version.ref = "edc" } +edc-edr-store-receiver = { module = "org.eclipse.edc:edr-store-receiver", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } edc-jersey-micrometer = { module = "org.eclipse.edc:jersey-micrometer", version.ref = "edc" } @@ -54,13 +58,13 @@ edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref edc-monitor-jdk-logger = { module = "org.eclipse.edc:monitor-jdk-logger", version.ref = "edc" } edc-provision-aws-s3 = { module = "org.eclipse.edc:provision-aws-s3", version.ref = "edc" } edc-runtime-metamodel = { module = "org.eclipse.edc:runtime-metamodel", version.ref = "edc" } -edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } +edc-transfer-data-plane-signaling = { module = "org.eclipse.edc:transfer-data-plane-signaling", version.ref = "edc" } edc-transfer-process-api = { module = "org.eclipse.edc:transfer-process-api", version.ref = "edc" } -edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } +edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-util = { module = "org.eclipse.edc:util", version.ref = "edc" } edc-vault-azure = { module = "org.eclipse.edc:vault-azure", version.ref = "edc" } -edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } +edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } jakartaJson = { module = "org.glassfish:jakarta.json", version.ref = "jakarta-json" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } @@ -69,7 +73,7 @@ junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", vers junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } okhttp-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp-mockwebserver" } opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version = "1.18.0" } -opentelemetry-exporter-jaeger = { module = "io.opentelemetry:opentelemetry-exporter-jaeger", version = "1.34.1" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version = "1.39.0" } opentelemetry-javaagent = { module = "io.opentelemetry.javaagent:opentelemetry-javaagent", version = "2.4.0" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index 82cabece..c9076e34 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -48,3 +48,7 @@ dependencies { testCompileOnly(project(":policy:policy-01-policy-enforcement:policy-enforcement-consumer")) testCompileOnly(project(":policy:policy-01-policy-enforcement:policy-functions")) } + +tasks.compileJava { + dependsOn(":advanced:advanced-01-open-telemetry:open-telemetry-runtime:build") +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java index 6566766e..f0571f16 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java @@ -39,7 +39,6 @@ import static org.eclipse.edc.samples.common.NegotiationCommon.fetchDatasetFromCatalog; import static org.eclipse.edc.samples.common.NegotiationCommon.getContractAgreementId; import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; @@ -67,14 +66,14 @@ static void setUp() { @Test void runSampleSteps() { - runPrerequisites(); createAsset(); createPolicy(); createContractDefinition(); var catalogDatasetId = fetchDatasetFromCatalog(FETCH_DATASET_FROM_CATALOG_FILE_PATH); var contractNegotiationId = negotiateContract(NEGOTIATE_CONTRACT_FILE_PATH, catalogDatasetId); var contractAgreementId = getContractAgreementId(contractNegotiationId); - var transferProcessId = startTransfer(getFileContentFromRelativePath(START_TRANSFER_FILE_PATH), contractAgreementId); + var transferRequest = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var transferProcessId = startTransfer(transferRequest, contractAgreementId); checkTransferStatus(transferProcessId, STARTED); assertJaegerState(); } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java index e365b91f..79055b21 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java @@ -14,13 +14,10 @@ package org.eclipse.edc.samples.common; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; import java.util.Map; -import static io.restassured.RestAssured.given; import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; public class PrerequisitesCommon { @@ -28,49 +25,34 @@ public class PrerequisitesCommon { public static final String API_KEY_HEADER_VALUE = "password"; public static final String PROVIDER_MANAGEMENT_URL = "http://localhost:19193/management"; public static final String CONSUMER_MANAGEMENT_URL = "http://localhost:29193/management"; - public static final String CONSUMER_PUBLIC_URL = "http://localhost:29291/public"; private static final String CONNECTOR_MODULE_PATH = ":transfer:transfer-00-prerequisites:connector"; private static final String PROVIDER = "provider"; private static final String CONSUMER = "consumer"; private static final String EDC_KEYSTORE = "edc.keystore"; private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password"; - private static final String EDC_VAULT = "edc.vault"; private static final String EDC_FS_CONFIG = "edc.fs.config"; private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx"; private static final String KEYSTORE_PASSWORD = "123456"; private static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties"; private static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties"; - private static final String PROVIDER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties"; - private static final String CONSUMER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties"; - private static final String REGISTER_DATA_PLANE_PROVIDER_JSON = "transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json"; public static EdcRuntimeExtension getProvider() { - return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_VAULT_PROPERTIES_FILE_PATH, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); - + return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); } public static EdcRuntimeExtension getConsumer() { - return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); } public static EdcRuntimeExtension getConsumer(String modulePath) { - return getConnector(modulePath, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); - } - - public static void registerDataPlaneProvider() { - registerDataPlane(PROVIDER_MANAGEMENT_URL, REGISTER_DATA_PLANE_PROVIDER_JSON); - } - - public static void runPrerequisites() { - registerDataPlaneProvider(); + return getConnector(modulePath, CONSUMER, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); } private static EdcRuntimeExtension getConnector( String modulePath, String moduleName, - String vaultPropertiesFilePath, String configPropertiesFilePath ) { return new EdcRuntimeExtension( @@ -79,24 +61,8 @@ private static EdcRuntimeExtension getConnector( Map.of( EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(), EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD, - EDC_VAULT, getFileFromRelativePath(vaultPropertiesFilePath).getAbsolutePath(), EDC_FS_CONFIG, getFileFromRelativePath(configPropertiesFilePath).getAbsolutePath() ) ); } - - private static void registerDataPlane(String host, String payloadPath) { - var requestBody = getFileFromRelativePath(payloadPath); - - given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(requestBody) - .when() - .post(host + "/v2/dataplanes") - .then() - .log() - .ifError() - .statusCode(HttpStatus.SC_OK); - } } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java deleted file mode 100644 index b05bb404..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Initial implementation - * - */ - -package org.eclipse.edc.samples.transfer; - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.registerDataPlaneProvider; - -@EndToEndTest -public class Transfer00prerequisitesTest { - - @RegisterExtension - static EdcRuntimeExtension provider = getProvider(); - - @RegisterExtension - static EdcRuntimeExtension consumer = getConsumer(); - - @Test - void runSampleSteps() { - registerDataPlaneProvider(); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java index 2e3bce76..1acc054f 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java @@ -29,7 +29,6 @@ import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; @EndToEndTest public class Transfer01negotiationTest { @@ -45,7 +44,6 @@ public class Transfer01negotiationTest { @Test void runSampleSteps() { - runPrerequisites(); createAsset(); createPolicy(); createContractDefinition(); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java index 1ba92337..91b9d8d9 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java @@ -15,18 +15,17 @@ package org.eclipse.edc.samples.transfer; +import io.restassured.common.mapper.TypeRef; import org.apache.http.HttpStatus; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcessStates; import org.eclipse.edc.junit.annotations.EndToEndTest; import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.eclipse.edc.samples.util.HttpRequestLoggerConsumer; -import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.util.Map; + import static io.restassured.RestAssured.given; import static org.apache.http.HttpHeaders.AUTHORIZATION; import static org.assertj.core.api.Assertions.assertThat; @@ -34,10 +33,9 @@ import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_PUBLIC_URL; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_MANAGEMENT_URL; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; import static org.hamcrest.Matchers.emptyString; @@ -47,9 +45,7 @@ @Testcontainers public class Transfer02consumerPullTest { - private static final HttpRequestLoggerConsumer LOG_CONSUMER = new HttpRequestLoggerConsumer(); private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-02-consumer-pull/resources/start-transfer.json"; - private static final String AUTH_CODE_KEY = "authCode"; @RegisterExtension static EdcRuntimeExtension provider = getProvider(); @@ -57,32 +53,30 @@ public class Transfer02consumerPullTest { @RegisterExtension static EdcRuntimeExtension consumer = getConsumer(); - @Container - public static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(LOG_CONSUMER); - - @BeforeAll - static void setUp() { - httpRequestLoggerContainer.start(); - } - @Test void runSampleSteps() { - runPrerequisites(); var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); var contractAgreementId = runNegotiation(); var transferProcessId = startTransfer(requestBody, contractAgreementId); checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); - var authCode = LOG_CONSUMER.getJsonValue(AUTH_CODE_KEY); + + var edr = given() + .when() + .get(CONSUMER_MANAGEMENT_URL + "/v1/edrs/{id}/dataaddress", transferProcessId) + .then() + .log().ifValidationFails() + .statusCode(200) + .extract().body().as(new TypeRef>() { + }); var result = given() .header(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .header(AUTHORIZATION, authCode) + .header(AUTHORIZATION, edr.get("authorization")) .when() - .get(CONSUMER_PUBLIC_URL) + .get(edr.get("endpoint").toString()) .then() .statusCode(HttpStatus.SC_OK) - .log() - .ifError() + .log().ifValidationFails() .body("[0].name", not(emptyString())) .extract() .jsonPath() diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java index 02e7af66..68c030a8 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java @@ -31,7 +31,6 @@ import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; @@ -58,7 +57,6 @@ static void setUp() { @Test void runSampleSteps() { - runPrerequisites(); var contractAgreementId = runNegotiation(); var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); var transferProcessId = startTransfer(requestBody, contractAgreementId); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java index 93c5c792..8c77f665 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java @@ -32,7 +32,6 @@ import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; -import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; @@ -59,7 +58,6 @@ static void setUp() { void runSampleSteps() { var standardOutputStream = new ByteArrayOutputStream(); System.setOut(new PrintStream(standardOutputStream)); - runPrerequisites(); var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); var contractAgreementId = runNegotiation(); var transferProcessId = startTransfer(requestBody, contractAgreementId); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java index 7f59cb28..f4c92958 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java @@ -28,7 +28,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -90,7 +89,6 @@ void setUp() throws IOException { @Test void streamData() throws IOException { var source = Files.createTempDirectory("source"); - PROVIDER.registerDataPlane(List.of("HttpStreaming"), List.of("HttpData"), List.of("HttpData-PUSH")); PROVIDER.createAsset(getFileContentFromRelativePath(SAMPLE_FOLDER + "/asset.json") .replace("{{sourceFolder}}", source.toString())); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java index 383b0014..8d03d0bd 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.net.URI; import java.time.Duration; -import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executors; @@ -108,9 +107,6 @@ void setUp() throws IOException { @Test void streamData() { - - PROVIDER.registerDataPlane(List.of("Kafka"), List.of("HttpData"), List.of("HttpData-PUSH")); - PROVIDER.createAsset(getFileContentFromRelativePath(SAMPLE_FOLDER + "/1-asset.json") .replace("{{bootstrap.servers}}", kafkaContainer.getBootstrapServers()) .replace("{{max.duration}}", MAX_DURATION) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/StreamingParticipant.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/StreamingParticipant.java index 274473d3..39627a8d 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/StreamingParticipant.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/StreamingParticipant.java @@ -14,19 +14,10 @@ package org.eclipse.edc.samples.transfer.streaming; -import jakarta.json.Json; import org.eclipse.edc.connector.controlplane.test.system.utils.Participant; -import java.util.List; -import java.util.UUID; - import static io.restassured.http.ContentType.JSON; -import static jakarta.json.Json.createArrayBuilder; -import static jakarta.json.Json.createObjectBuilder; -import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.CONTEXT; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; -import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; -import static org.eclipse.edc.spi.constants.CoreConstants.EDC_PREFIX; /** * Essentially a wrapper around the management API enabling to test interactions with other participants, eg. catalog, transfer... @@ -42,25 +33,6 @@ public String getName() { return name; } - public void registerDataPlane(List sourceTypes, List destinationTypes, List transferTypes) { - var jsonObject = Json.createObjectBuilder() - .add(CONTEXT, createObjectBuilder().add(EDC_PREFIX, EDC_NAMESPACE)) - .add(ID, UUID.randomUUID().toString()) - .add(EDC_NAMESPACE + "url", controlEndpoint.getUrl() + "/transfer") - .add(EDC_NAMESPACE + "allowedSourceTypes", createArrayBuilder(sourceTypes)) - .add(EDC_NAMESPACE + "allowedDestTypes", createArrayBuilder(destinationTypes)) - .add(EDC_NAMESPACE + "allowedTransferTypes", createArrayBuilder(transferTypes)) - .build(); - - managementEndpoint.baseRequest() - .contentType(JSON) - .body(jsonObject.toString()) - .when() - .post("/v2/dataplanes") - .then() - .statusCode(200); - } - public String createAsset(String requestBody) { return managementEndpoint.baseRequest() .contentType(JSON) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java index 4f443640..802bcbb8 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java @@ -22,6 +22,7 @@ import java.time.Duration; import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; @@ -104,9 +105,9 @@ public static void checkTransferStatus(String transferProcessId, TransferProcess .atMost(TIMEOUT) .pollDelay(POLL_DELAY) .pollInterval(POLL_INTERVAL) - .until( - () -> get(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH + transferProcessId, EDC_STATE), - (result) -> status.name().equals(result) - ); + .untilAsserted(() -> { + var state = get(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH + transferProcessId, EDC_STATE); + assertThat(state).isEqualTo(status.name()); + }); } } diff --git a/transfer/streaming/streaming-01-http-to-http/README.md b/transfer/streaming/streaming-01-http-to-http/README.md index 66331433..169cff02 100644 --- a/transfer/streaming/streaming-01-http-to-http/README.md +++ b/transfer/streaming/streaming-01-http-to-http/README.md @@ -26,15 +26,6 @@ export EDC_FS_CONFIG=transfer/streaming/streaming-01-http-to-http/streaming-01-r java -jar transfer/streaming/streaming-01-http-to-http/streaming-01-runtime/build/libs/connector.jar ``` -#### Register Data Plane on provider -The provider connector needs to be aware of the streaming capabilities of the embedded dataplane, which can be registered with -this call: -```shell -curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-01-http-to-http/dataplane.json -X POST "http://localhost:18181/management/v2/dataplanes" -``` - -If you look at the `dataplane.json` you'll notice that the supported source is `HttpStreaming` and the supported sink is `HttpData`. - #### Register Asset, Policy Definition and Contract Definition on provider A "source" folder must first be created where the data plane will get the messages to be sent to the consumers. To do this, create a temp folder: diff --git a/transfer/streaming/streaming-01-http-to-http/dataplane.json b/transfer/streaming/streaming-01-http-to-http/dataplane.json deleted file mode 100644 index a7980a42..00000000 --- a/transfer/streaming/streaming-01-http-to-http/dataplane.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ "HttpStreaming" ], - "allowedDestTypes": [ "HttpData" ] -} diff --git a/transfer/streaming/streaming-01-http-to-http/streaming-01-runtime/build.gradle.kts b/transfer/streaming/streaming-01-http-to-http/streaming-01-runtime/build.gradle.kts index a3ddd8b5..03f300d8 100644 --- a/transfer/streaming/streaming-01-http-to-http/streaming-01-runtime/build.gradle.kts +++ b/transfer/streaming/streaming-01-http-to-http/streaming-01-runtime/build.gradle.kts @@ -30,9 +30,10 @@ dependencies { implementation(libs.edc.management.api) implementation(libs.edc.dsp) implementation(libs.edc.data.plane.selector.api) - implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.data.plane.spi) implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.self.registration) implementation(libs.edc.data.plane.http) } diff --git a/transfer/streaming/streaming-02-kafka-to-http/README.md b/transfer/streaming/streaming-02-kafka-to-http/README.md index c404fef6..9ded22d2 100644 --- a/transfer/streaming/streaming-02-kafka-to-http/README.md +++ b/transfer/streaming/streaming-02-kafka-to-http/README.md @@ -27,16 +27,6 @@ export EDC_FS_CONFIG=transfer/streaming/streaming-02-kafka-to-http/streaming-02- java -jar transfer/streaming/streaming-02-kafka-to-http/streaming-02-runtime/build/libs/connector.jar ``` -### Register Data Plane on provider - -The provider connector needs to be aware of the kafka streaming capabilities of the embedded dataplane, which can be registered with -this call: -```shell -curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-02-kafka-to-http/0-dataplane.json -X POST "http://localhost:18181/management/v2/dataplanes" -s | jq -``` - -If you look at the `0-dataplane.json` you'll notice that the supported source is `Kafka` and the supported sink is `HttpData`. - ### Register Asset, Policy Definition and Contract Definition on provider A "source" kafka topic must first be created where the data plane will get the event records to be sent to the consumers. diff --git a/transfer/streaming/streaming-02-kafka-to-http/streaming-02-runtime/build.gradle.kts b/transfer/streaming/streaming-02-kafka-to-http/streaming-02-runtime/build.gradle.kts index e8721d84..fa662fd7 100644 --- a/transfer/streaming/streaming-02-kafka-to-http/streaming-02-runtime/build.gradle.kts +++ b/transfer/streaming/streaming-02-kafka-to-http/streaming-02-runtime/build.gradle.kts @@ -30,9 +30,10 @@ dependencies { implementation(libs.edc.management.api) implementation(libs.edc.dsp) implementation(libs.edc.data.plane.selector.api) - implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.data.plane.spi) implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.self.registration) implementation(libs.edc.data.plane.http) implementation(libs.edc.data.plane.kafka) diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts index 8b6a0914..537c3e85 100644 --- a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts @@ -28,7 +28,7 @@ dependencies { implementation(libs.edc.management.api) implementation(libs.edc.dsp) implementation(libs.edc.data.plane.selector.api) - implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.dynamic.receiver) implementation(libs.edc.data.plane.spi) implementation(libs.edc.data.plane.core) diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md index b79ddad0..e0820e92 100644 --- a/transfer/transfer-00-prerequisites/README.md +++ b/transfer/transfer-00-prerequisites/README.md @@ -40,57 +40,24 @@ Inspect the different configuration files below: * [provider-configuration.properties](resources/configuration/provider-configuration.properties) * [consumer-configuration.properties](resources/configuration/consumer-configuration.properties) -The section bellow will show you some explanation about some of the properties that you can find in -the configuration files. - -#### 1. edc.receiver.http.endpoint - -This property is used to define the endpoint where the connector consumer will send the -EndpointDataReference. - -#### 2. edc.dataplane.token.validation.endpoint - -This property is used to define the endpoint exposed by the control plane to validate the token. - ### 2. Run the connectors To run the provider, just run the following command ```bash -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar ``` To run the consumer, just run the following command (different terminal) ```bash -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar ``` Assuming you didn't change the ports in config files, the consumer will listen on the ports `29191`, `29192` (management API) and `29292` (DSP API) and the provider will listen on the ports `12181`, `19182` (management API) and `19282` (DSP API). -Running this sample consists of multiple steps, that are executed one by one and following the same -order. - -### 3. Register data plane instance for provider - -Before a consumer can start talking to a provider, it is necessary to register the data plane -instance of a connector. This is done by sending a POST request to the management API of the -provider connector. The [request body](resources/dataplane/register-data-plane-provider.json) should contain the data plane instance of the consumer -connector. - -The registration of the provider data plane instance is done by sending a POST -request to the management API of the connector. - -Open a new terminal and execute: - -```bash -curl -H 'Content-Type: application/json' \ - -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ - -X POST "http://localhost:19193/management/v2/dataplanes" -s | jq -``` - The connectors have been configured successfully and are ready to be used. [Next Chapter](../transfer-01-negotiation/README.md) diff --git a/transfer/transfer-00-prerequisites/connector/build.gradle.kts b/transfer/transfer-00-prerequisites/connector/build.gradle.kts index 3ce117a2..36c0d111 100644 --- a/transfer/transfer-00-prerequisites/connector/build.gradle.kts +++ b/transfer/transfer-00-prerequisites/connector/build.gradle.kts @@ -24,15 +24,20 @@ dependencies { implementation(libs.edc.control.plane.core) implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.vault.filesystem) implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) - implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.receiver) + implementation(libs.edc.validator.data.address.http.data) + + implementation(libs.edc.edr.cache.api) + implementation(libs.edc.edr.store.core) + implementation(libs.edc.edr.store.receiver) implementation(libs.edc.data.plane.selector.api) implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.self.registration) implementation(libs.edc.data.plane.control.api) implementation(libs.edc.data.plane.public.api) implementation(libs.edc.data.plane.core) diff --git a/transfer/transfer-00-prerequisites/connector/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java b/transfer/transfer-00-prerequisites/connector/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java new file mode 100644 index 00000000..171d96ff --- /dev/null +++ b/transfer/transfer-00-prerequisites/connector/src/main/java/org/eclipse/edc/sample/runtime/SeedVaultExtension.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.sample.runtime; + +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +public class SeedVaultExtension implements ServiceExtension { + + @Inject + private Vault vault; + + private static final String PUBLIC_KEY = """ + -----BEGIN CERTIFICATE----- + MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL + BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM + GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy + MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw + HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB + AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g + E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS + PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H + I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W + EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 + h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud + DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B + 2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn + QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ + rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe + Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy + +NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR + IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ + g3h+15GuzbsSzOCOEYOT + -----END CERTIFICATE----- + """; + + private static final String PRIVATE_KEY = """ + -----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBl6XaJnXTL+6D + Wip3aBhU+MzmY4d1V9hbTm1tiZ3gE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7 + EddidN0ITHB9cQNdAfdUJ5njmsGSPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7 + DHacZT/+OztBH1RwkG2ymM94Hf8HI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvj + X5qASakBtXISKIsOU84N0/2HDN3WEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga7 + 75bPXN3M+JTSaIKE7dZbKzvx0Zi0h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2 + YmnneYoVAgMBAAECggEBAJHXiN6bctAyn+DcoHlsNkhtVw+Jk5bXIutGXjHTJtiU + K//siAGC78IZMyXmi0KndPVCdBwShROVW8xWWIiXuZxy2Zvm872xqX4Ah3JsN7/Q + NrXdVBUDo38zwIGkxqIfIz9crZ4An+J/eq5zaTfRHzCLtswMqjRS2hFeBY5cKrBY + 4bkSDGTP/c5cP7xS/UwaiTR2Ptd41f4zTyd4l5rl30TYHpazQNlbdxcOV4jh2Rnp + E0+cFEvEfeagVq7RmfBScKG5pk4qcRG0q2QHMyK5y00hdYvhdRjSgN7xIDkeO5B8 + s8/tSLU78nCl2gA9IKxTXYLitpISwZ81Q04mEAKRRtECgYEA+6lKnhn//aXerkLo + ZOLOjWQZhh005jHdNxX7DZqLpTrrfxc8v15KWUkAK1H0QHqYvfPrbbsBV1MY1xXt + sKmkeu/k8fJQzCIvFN4K2J5W5kMfq9PSw5d3XPeDaQuXUVaxBVp0gzPEPHmkKRbA + AkUqY0oJwA9gMKf8dK+flmLZfbsCgYEAxO4Roj2G46/Oox1GEZGxdLpiMpr9rEdR + JlSZ9kMGfddNLV7sFp6yPXDcyc/AOqeNj7tw1MyoT3Ar454+V0q83EZzCXvs4U6f + jUrfFcoVWIwf9AV/J4KWzMIzfqPIeNwqymZKd6BrZgcXXvAEPWt27mwO4a1GhC4G + oZv0t3lAsm8CgYAQ8C0IhSF4tgBN5Ez19VoHpDQflbmowLRt77nNCZjajyOokyzQ + iI0ig0pSoBp7eITtTAyNfyew8/PZDi3IVTKv35OeQTv08VwP4H4EZGve5aetDf3C + kmBDTpl2qYQOwnH5tUPgTMypcVp+NXzI6lTXB/WuCprjy3qvc96e5ZpT3wKBgQC8 + Xny/k9rTL/eYTwgXBiWYYjBL97VudUlKQOKEjNhIxwkrvQBXIrWbz7lh0Tcu49al + BcaHxru4QLO6pkM7fGHq0fh3ufJ8EZjMrjF1xjdk26Q05o0aXe+hLKHVIRVBhlfo + ArB4fRo+HcpdJXjox0KcDQCvHe+1v9DYBTWvymv4QQKBgBy3YH7hKz35DcXvA2r4 + Kis9a4ycuZqTXockO4rkcIwC6CJp9JbHDIRzig8HYOaRqmZ4a+coqLmddXr2uOF1 + 7+iAxxG1KzdT6uFNd+e/j2cdUjnqcSmz49PRtdDswgyYhoDT+W4yVGNQ4VuKg6a3 + Z3pC+KTdoHSKeA2FyAGnSUpD + -----END PRIVATE KEY----- + """; + + @Override + public void initialize(ServiceExtensionContext context) { + vault.storeSecret("public-key", PUBLIC_KEY); + vault.storeSecret("private-key", PRIVATE_KEY); + } +} diff --git a/transfer/transfer-00-prerequisites/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-00-prerequisites/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..eaca6845 --- /dev/null +++ b/transfer/transfer-00-prerequisites/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.runtime.SeedVaultExtension diff --git a/transfer/transfer-00-prerequisites/resources/certs/cert.pem b/transfer/transfer-00-prerequisites/resources/certs/cert.pem deleted file mode 100644 index c7dc26fa..00000000 --- a/transfer/transfer-00-prerequisites/resources/certs/cert.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy -MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g -E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS -PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H -I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W -EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 -h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud -DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B -2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn -QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ -rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe -Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy -+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR -IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ -g3h+15GuzbsSzOCOEYOT ------END CERTIFICATE----- diff --git a/transfer/transfer-00-prerequisites/resources/certs/key.pem b/transfer/transfer-00-prerequisites/resources/certs/key.pem deleted file mode 100644 index e72229e8..00000000 --- a/transfer/transfer-00-prerequisites/resources/certs/key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBl6XaJnXTL+6D -Wip3aBhU+MzmY4d1V9hbTm1tiZ3gE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7 -EddidN0ITHB9cQNdAfdUJ5njmsGSPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7 -DHacZT/+OztBH1RwkG2ymM94Hf8HI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvj -X5qASakBtXISKIsOU84N0/2HDN3WEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga7 -75bPXN3M+JTSaIKE7dZbKzvx0Zi0h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2 -YmnneYoVAgMBAAECggEBAJHXiN6bctAyn+DcoHlsNkhtVw+Jk5bXIutGXjHTJtiU -K//siAGC78IZMyXmi0KndPVCdBwShROVW8xWWIiXuZxy2Zvm872xqX4Ah3JsN7/Q -NrXdVBUDo38zwIGkxqIfIz9crZ4An+J/eq5zaTfRHzCLtswMqjRS2hFeBY5cKrBY -4bkSDGTP/c5cP7xS/UwaiTR2Ptd41f4zTyd4l5rl30TYHpazQNlbdxcOV4jh2Rnp -E0+cFEvEfeagVq7RmfBScKG5pk4qcRG0q2QHMyK5y00hdYvhdRjSgN7xIDkeO5B8 -s8/tSLU78nCl2gA9IKxTXYLitpISwZ81Q04mEAKRRtECgYEA+6lKnhn//aXerkLo -ZOLOjWQZhh005jHdNxX7DZqLpTrrfxc8v15KWUkAK1H0QHqYvfPrbbsBV1MY1xXt -sKmkeu/k8fJQzCIvFN4K2J5W5kMfq9PSw5d3XPeDaQuXUVaxBVp0gzPEPHmkKRbA -AkUqY0oJwA9gMKf8dK+flmLZfbsCgYEAxO4Roj2G46/Oox1GEZGxdLpiMpr9rEdR -JlSZ9kMGfddNLV7sFp6yPXDcyc/AOqeNj7tw1MyoT3Ar454+V0q83EZzCXvs4U6f -jUrfFcoVWIwf9AV/J4KWzMIzfqPIeNwqymZKd6BrZgcXXvAEPWt27mwO4a1GhC4G -oZv0t3lAsm8CgYAQ8C0IhSF4tgBN5Ez19VoHpDQflbmowLRt77nNCZjajyOokyzQ -iI0ig0pSoBp7eITtTAyNfyew8/PZDi3IVTKv35OeQTv08VwP4H4EZGve5aetDf3C -kmBDTpl2qYQOwnH5tUPgTMypcVp+NXzI6lTXB/WuCprjy3qvc96e5ZpT3wKBgQC8 -Xny/k9rTL/eYTwgXBiWYYjBL97VudUlKQOKEjNhIxwkrvQBXIrWbz7lh0Tcu49al -BcaHxru4QLO6pkM7fGHq0fh3ufJ8EZjMrjF1xjdk26Q05o0aXe+hLKHVIRVBhlfo -ArB4fRo+HcpdJXjox0KcDQCvHe+1v9DYBTWvymv4QQKBgBy3YH7hKz35DcXvA2r4 -Kis9a4ycuZqTXockO4rkcIwC6CJp9JbHDIRzig8HYOaRqmZ4a+coqLmddXr2uOF1 -7+iAxxG1KzdT6uFNd+e/j2cdUjnqcSmz49PRtdDswgyYhoDT+W4yVGNQ4VuKg6a3 -Z3pC+KTdoHSKeA2FyAGnSUpD ------END PRIVATE KEY----- diff --git a/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties index bb7df005..5b2059ad 100644 --- a/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties +++ b/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties @@ -6,13 +6,9 @@ web.http.management.port=29193 web.http.management.path=/management web.http.protocol.port=29194 web.http.protocol.path=/protocol -edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback -edc.public.key.alias=public-key -edc.transfer.dataplane.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.signer.privatekey.alias=1 +edc.transfer.proxy.token.signer.privatekey.alias=private-key edc.transfer.proxy.token.verifier.publickey.alias=public-key web.http.public.port=29291 web.http.public.path=/public web.http.control.port=29192 web.http.control.path=/control -edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token diff --git a/transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties deleted file mode 100644 index 6ebdebd5..00000000 --- a/transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties +++ /dev/null @@ -1 +0,0 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- diff --git a/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties index 4679ff53..733d735a 100644 --- a/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties +++ b/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties @@ -6,13 +6,10 @@ web.http.management.port=19193 web.http.management.path=/management web.http.protocol.port=19194 web.http.protocol.path=/protocol -edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback -edc.public.key.alias=public-key -edc.transfer.dataplane.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.signer.privatekey.alias=1 +edc.transfer.proxy.token.signer.privatekey.alias=private-key edc.transfer.proxy.token.verifier.publickey.alias=public-key web.http.public.port=19291 web.http.public.path=/public web.http.control.port=19192 web.http.control.path=/control -edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token +edc.dataplane.api.public.baseurl=http://localhost:19291/public diff --git a/transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties deleted file mode 100644 index 6ebdebd5..00000000 --- a/transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties +++ /dev/null @@ -1 +0,0 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json deleted file mode 100644 index 7ca6ff1a..00000000 --- a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-consumer-dataplane", - "url": "http://localhost:29192/control/transfer", - "allowedSourceTypes": [ - "HttpData" - ], - "allowedDestTypes": [ - "HttpProxy", - "HttpData" - ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" - } -} diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json deleted file mode 100644 index 0ee24de4..00000000 --- a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ - "HttpData" - ], - "allowedDestTypes": [ - "HttpProxy", - "HttpData" - ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } -} diff --git a/transfer/transfer-02-consumer-pull/README.md b/transfer/transfer-02-consumer-pull/README.md index 8df9f52a..0c8706cb 100644 --- a/transfer/transfer-02-consumer-pull/README.md +++ b/transfer/transfer-02-consumer-pull/README.md @@ -23,17 +23,7 @@ and [Negotiation](../transfer-01-negotiation/README.md) chapters. Running this sample consists of multiple steps, that are executed one by one and following the same order. -### 1. Start a http server - -As a pre-requisite, you need to have a logging webserver that runs on port 4000 and logs all the incoming requests, it will -be mandatory to get the EndpointDataReference that will be used to get the data. - -```bash -docker build -t http-request-logger util/http-request-logger -docker run -p 4000:4000 http-request-logger -``` - -### 2. Start the transfer +### 1. Start the transfer In the [request body](resources/start-transfer.json), we need to specify which asset we want transferred, the ID of the contract agreement, the address of the provider connector and where we want the file transferred. @@ -47,9 +37,8 @@ curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ ``` -> the "HttpProxy" method is used for the consumer pull method, and it means that it will be up to -> the consumer to request the data to the provider and that the request will be a proxy for the -> datasource +> the "HttpData-PULL" transfer type is used for the consumer pull method, and it means that it will be up to +> the consumer to request the data to the provider and that the request will be a proxy for the datasource Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( process id) created on the consumer @@ -67,7 +56,7 @@ Sample output: } ``` -### 3. Check the transfer status +### 2. Check the transfer status Due to the nature of the transfer, it will be very fast and most likely already done by the time you read the UUID. @@ -91,28 +80,36 @@ You should see the Transfer Process in `STARTED` state: > Note that for the consumer pull scenario the TP will stay in STARTED state after the data has been transferred successfully. > It might get eventually get shifted to TERMINATED or DEPROVISIONED by other resources, but this is not scope of this sample. -### 4. Check the data +### 3. Check the data -At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed -to get the data from the provider: +At this step, an EndpointDataReference would have been generated by the provider and sent to the consumer. The latter +stored it in a cache, so we can obtain it using the transfer process id: +```bash +curl http://localhost:29193/management/v1/edrs//dataaddress | jq +``` ```json { - "id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "endpoint": "http://localhost:19291/public/", - "authKey": "Authorization", - "authCode": "{{auth-code}}", - "properties": { - "cid": "1:1e47895f-9025-49e1-971d-eb865c38d540" + "@type": "DataAddress", + "type": "https://w3id.org/idsa/v4.1/HTTP", + "endpoint": "http://localhost:19291/public", + "authType": "bearer", + "endpointType": "https://w3id.org/idsa/v4.1/HTTP", + "authorization": "eyJraWQiOiJwdWJsaWMta2V5IiwiYWxnIjoiUlMyNTYifQ.eyJpc3MiOiJwcm92aWRlciIsImF1ZCI6ImNvbnN1bWVyIiwic3ViIjoicHJvdmlkZXIiLCJpYXQiOjE3MTc3NjkyMzEyOTYsImp0aSI6IjM2M2RhMGU4LWZmOGItNDY1My05YjQwLWY4MjdlMWMzOGMzYyJ9.WOVPz6m7XzIrbiMTfLqOXacGYz8Xk_-iQu7gmxoIgDFYsgo0da2Iv51EsugIpqbodPsmB0kK7zkyrmsFOfAASAq7fjsy4gQF-u5egYwoGpcxjYaJJdQa5lkwjC0fRxdVFVwZwrOaT5Mg-vGA9HssTEnlA64q-O0ae_aTH5ToflmPDM3FhAgL55I3odM5ysM2POEJY6pgOxIV9XjuhZFl_i_iTiUCZy__oQUZiYk58wKoqfK758Sy1WzpH-eyZCDUi_Z3n6cJB80_0ZThoPhtiFH7Tl9DfStnjsCoaeqMLFnTXp0s8h4ZGFmjfBc-72aAdRQqqLDT8WXNg3Csv5B56Q", + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" } } + ``` Once this json is read, use a tool like postman or curl to execute the following query, to read the data ```bash -curl --location --request GET 'http://localhost:19291/public/' --header 'Authorization: ' +curl --location --request GET 'http://localhost:19291/public/' --header 'Authorization: ' ``` At the end, and to be sure that you correctly achieved the pull, you can check if the data you get diff --git a/transfer/transfer-03-provider-push/README.md b/transfer/transfer-03-provider-push/README.md index 495c3fef..98216af4 100644 --- a/transfer/transfer-03-provider-push/README.md +++ b/transfer/transfer-03-provider-push/README.md @@ -24,7 +24,17 @@ If not, re-visit the [Prerequisites](../transfer-00-prerequisites/README.md) Running this sample consists of multiple steps, that are executed one by one and following the same order. -### 1. Start the transfer +### 1. Start a http server + +As a pre-requisite, you need to have a logging webserver that runs on port 4000 and logs all the incoming requests, the +data will be sent to this server. + +```bash +docker build -t http-request-logger util/http-request-logger +docker run -p 4000:4000 http-request-logger +``` + +### 2. Start the transfer Before executing the request, modify the [request body](resources/start-transfer.json) by inserting the contract agreement ID from the [Negotiation](../transfer-01-negotiation/README.md) chapter. @@ -50,7 +60,7 @@ Sample output: } ``` -### 2. Check the transfer status +### 3. Check the transfer status Due to the nature of the transfer, it will be very fast and most likely already done by the time you read the UUID. @@ -61,7 +71,7 @@ curl http://localhost:29193/management/v2/transferprocesses/