From 19d0483ceb448c7f5c0d6c0ca01fdd3831e66faf Mon Sep 17 00:00:00 2001 From: farhin23 Date: Tue, 21 Mar 2023 10:07:52 +0100 Subject: [PATCH 01/35] feat: create basic policy sample --- .../policy-01-contract-negotiation/README.md | 147 ++++++++++++++++++ .../contractoffer.json | 70 +++++++++ .../build.gradle.kts | 34 ++++ .../config.properties | 14 ++ .../build.gradle.kts | 14 ++ .../policy/PolicyFunctionsExtension.java | 143 +++++++++++++++++ .../policy/TimeIntervalFunction.java | 48 ++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + settings.gradle.kts | 5 + 9 files changed, 476 insertions(+) create mode 100644 policy/policy-01-contract-negotiation/README.md create mode 100644 policy/policy-01-contract-negotiation/contractoffer.json create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/policy/policy-01-contract-negotiation/README.md b/policy/policy-01-contract-negotiation/README.md new file mode 100644 index 00000000..5778a1da --- /dev/null +++ b/policy/policy-01-contract-negotiation/README.md @@ -0,0 +1,147 @@ +# Create a policy function + +This sample shows how a custom function for policy evaluation can be implemented and registered. For simplicity, +we will use just one connector, which initiates a contract negotiation with itself. The sample contains two modules: + +- `policy-contract-negotiation-connector`: contains the connector configuration +- `policy-contract-negotiation-policy-functions`: provides the function for policy evaluation + +## Creating and registering the function + +The [`TimeIntervalFunction`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java) +implements the `AtomicConstraintFunction` interface, which contains a single method for evaluating a constraint. +In that method, the `operator` and `right value` of the constraint can be used for evaluation. In this example, we +want to check whether the evaluation happens within a specified time interval. So depending on whether the operator is +`GT` or `LT`, we check if the current date is before or after the date specified in the `right value`. + +```java +@Override +public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + Date date; + try { + date = sdf.parse((String) rightValue); + } catch (ParseException e) { + monitor.severe("Failed to parse right value of constraint to date."); + return false; + } + + switch (operator) { + case LT: var isBefore = new Date().before(date); + monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date."); + return isBefore; + case GT: var isAfter = new Date().after(date); + monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date."); + return isAfter; + default: return false; + } +} +``` + +Next, we have to register our function with the `PolicyEngine` and bind the desired action as well as the key used to +register our function to the desired scopes using the `RuleBindingRegistry`. This is done in the +[`PolicyFunctionsExtension`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java): + +```java +private final String policyTimeKey = "POLICY_EVALUATION_TIME"; + +//... + +@Override +public void initialize(ServiceExtensionContext context) { + //... + + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); + policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); + + //... +} +``` + +## Defining the policy + +Next to registering the function and creating the binding, the `PolicyFunctionsExtension` also creates an asset, +a policy and a contract definition linking the former two. The asset is the same one that is used in +sample [`transfer-01-file-transfer`](transfer/transfer-01-file-transfer/README.md), but this time a different policy is created. + +We start by creating the constraints, where we define that the policy evaluation time must be within a certain +time interval. We read the start and end dates for the interval from the settings. Then, we create two constraints: +one specifying that the evaluation time should be after our defined start date, and one specifying that the evaluation +time should be before our defined end date. **We need to set the constraints' left operands to the same key +that we previously used to register our function. Otherwise, the function will not be used to evaluate these +constraints.** + +```java +var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); +var notBeforeConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.GT) + .rightExpression(new LiteralExpression(startDate)) + .build(); + +var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T23:59:00.000+02:00"); +var notAfterConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.LT) + .rightExpression(new LiteralExpression(endDate)) + .build(); +``` + +Then, we create a `Permission` with action type `USE` and the two constraints. We use this permission to create and +store a policy. And last, we create the `ContractDefinition`. For the access policy, we use the same use-policy that is +used in sample `transfer:transfer-01-file-transfer`, and for the contract policy, we use the previously created policy with the time +interval restriction. We set the `AssetSelectorExpression` so that the contract definition is valid for our asset. + +## How to run the sample + +To see the policy enforcement working, this sample should be run twice with different configurations. + +### Configuration + +Choose one of the following configurations. Depending on which configuration you choose, the contract negotiation +will either be confirmed or declined. + +#### 1. Policy fulfilled + +Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is within +the defined interval. + +```properties +edc.samples.uc.constraint.date.start=2023-01-01T00:00:00.000+02:00 +edc.samples.uc.constraint.date.end=2023-12-31T23:59:00.000+02:00 +``` + +#### 2. Policy not fulfilled + +Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is +**not** within the defined interval. + +```properties +edc.samples.policy-01.constraint.date.start=2022-01-01T00:00:00.000+02:00 +edc.samples.policy-01.constraint.date.end=2022-12-31T23:59:00.000+02:00 +``` + +### Run the sample + +First, build and run the connector for this sample: + +```shell +./gradlew policy:policy-01-contract-negotiation:policy-contract-negotiation-connector:build +java -Dedc.fs.config=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar +#for windows: +java -D"edc.fs.config"=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar +``` + +Next, initiate a contract negotiation. The request body is prepared in [`contractoffer.json`](policy/policy-01-contract-negotiation/contractoffer.json). +In that file, replace the dates for the constraints with the dates you used in the `config.properties` file and then +run: + +```shell +curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @samples/uc-workshop/contractoffer.json "http://localhost:8182/api/v1/management/contractnegotiations" +``` + +Depending on the configuration you chose, you can see the contract negotiation either being confirmed or declined +in the connector logs. The logs also contain messages logged by the `TimeIntervalFunction`, which tell whether the +current date is before or after the start and end dates defined in the policy. diff --git a/policy/policy-01-contract-negotiation/contractoffer.json b/policy/policy-01-contract-negotiation/contractoffer.json new file mode 100644 index 00000000..9af961ad --- /dev/null +++ b/policy/policy-01-contract-negotiation/contractoffer.json @@ -0,0 +1,70 @@ +{ + "connectorId": "provider", + "connectorAddress": "http://localhost:8282/api/v1/ids/data", + "protocol": "ids-multipart", + "offer": { + "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "assetId": "test-document", + "policy": { + "uid": "956e172f-2de1-4501-8881-057a57fd0e69", + "permissions": [ + { + "edctype": "dataspaceconnector:permission", + "uid": null, + "target": "test-document", + "action": { + "type": "USE", + "includedIn": null, + "constraint": null + }, + "assignee": null, + "assigner": null, + "constraints": [ + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2023-01-01T00:00:00.000+02:00" + }, + "operator": "GT" + }, + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2023-12-31T00:00:00.000+02:00" + }, + "operator": "LT" + } + ], + "duties": [] + } + ], + "prohibitions": [], + "obligations": [], + "extensibleProperties": {}, + "inheritsFrom": null, + "assigner": null, + "assignee": null, + "target": null, + "@type": { + "@policytype": "set" + } + }, + "asset": { + "properties": { + "ids:byteSize": null, + "asset:prop:id": "test-document", + "ids:fileName": null + } + } + } +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts new file mode 100644 index 00000000..a60d9218 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts @@ -0,0 +1,34 @@ +plugins { + `java-library` + id("application") + id("com.github.johnrengelman.shadow") version "7.1.2" +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + implementation("$groupId:control-plane-core:$edcVersion") + + implementation("$groupId:api-observability:$edcVersion") + + implementation("$groupId:configuration-filesystem:$edcVersion") + implementation("$groupId:iam-mock:$edcVersion") + + implementation("$groupId:auth-tokenbased:$edcVersion") + implementation("$groupId:management-api:$edcVersion") + + implementation("$groupId:ids:$edcVersion") + + implementation(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + exclude("**/pom.properties", "**/pom.xm") + mergeServiceFiles() + archiveFileName.set("connector.jar") +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties new file mode 100644 index 00000000..4a9f967f --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties @@ -0,0 +1,14 @@ +web.http.port=8181 +web.http.path=/api + +web.http.management.port=8182 +web.http.management.path=/api/v1/management + +edc.api.auth.key=password +ids.webhook.address=http://localhost:8282 +edc.ids.id=urn:connector:provider + +edc.samples.policy-01.asset.path = /path/to/file + +edc.samples.policy-01.constraint.date.start=2023-01-01T00:00:00.000+02:00 +edc.samples.policy-01.constraint.date.end=2023-12-31T00:00:00.000+02:00 diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts new file mode 100644 index 00000000..df01ec91 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` + id("application") +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + api("$groupId:data-plane-spi:$edcVersion") + + implementation("$groupId:control-plane-core:$edcVersion") + +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java new file mode 100644 index 00000000..a2dca9f6 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -0,0 +1,143 @@ +package org.eclipse.edc.sample.extension.policy; + +import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; +import org.eclipse.edc.connector.contract.spi.types.offer.ContractDefinition; +import org.eclipse.edc.connector.policy.spi.PolicyDefinition; +import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; +import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; +import org.eclipse.edc.policy.model.*; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.asset.AssetIndex; +import org.eclipse.edc.spi.asset.AssetSelectorExpression; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.asset.Asset; + +import java.nio.file.Path; + +import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; + +public class PolicyFunctionsExtension implements ServiceExtension { + private final String policyTimeKey = "POLICY_EVALUATION_TIME"; + private final String policyStartDateSetting = "edc.samples.policy-01.constraint.date.start"; + private final String policyEndDateSetting = "edc.samples.policy-01.constraint.date.end"; + private final String EDC_ASSET_PATH = "edc.samples.policy-01.asset.path"; + + @Inject + private RuleBindingRegistry ruleBindingRegistry; + + @Inject + private PolicyEngine policyEngine; + + @Inject + private PolicyDefinitionStore policyStore; + + @Inject + private ContractDefinitionStore contractDefinitionStore; + + @Inject + private AssetIndex assetIndex; + + @Override + public String name() { + return "Policy - contract-negotiation policies"; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); + policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); + + registerDataEntries(context); + registerContractDefinition(context); + + context.getMonitor().info("Policy Extension for Policy Sample (contract-negotiation) initialized!"); + } + + private PolicyDefinition createAccessPolicy() { + + var usePermission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance().type("USE").build()) + .build(); + + return PolicyDefinition.Builder.newInstance() + .id("use") + .policy(Policy.Builder.newInstance() + .permission(usePermission) + .build()) + .build(); + } + + private PolicyDefinition createContractPolicy(ServiceExtensionContext context) { + var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); + var notBeforeConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.GT) + .rightExpression(new LiteralExpression(startDate)) + .build(); + + var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T00:00:00.000+02:00"); + var notAfterConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.LT) + .rightExpression(new LiteralExpression(endDate)) + .build(); + + + var permission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance().type("USE").build()) + .constraint(notBeforeConstraint) + .constraint(notAfterConstraint) + .build(); + + + return PolicyDefinition.Builder.newInstance() + .id("use-time-restricted") + .policy(Policy.Builder.newInstance() + .permission(permission) + .build()) + .build(); + } + + + private void registerDataEntries(ServiceExtensionContext context) { + var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); + var assetPath = Path.of(assetPathSetting); + + var dataAddress = DataAddress.Builder.newInstance() + .property("type", "File") + .property("path", assetPath.getParent().toString()) + .property("filename", assetPath.getFileName().toString()) + .build(); + + var assetId = "test-document"; + var asset = Asset.Builder.newInstance().id(assetId).build(); + + assetIndex.accept(asset, dataAddress); + } + + private void registerContractDefinition(ServiceExtensionContext context) { + var accessPolicy = createAccessPolicy(); + policyStore.save(accessPolicy); + + var contractPolicy = createContractPolicy(context); + policyStore.save(contractPolicy); + + var contractDefinition = ContractDefinition.Builder.newInstance() + .id("1") + .accessPolicyId(accessPolicy.getUid()) + .contractPolicyId(contractPolicy.getUid()) + .selectorExpression(AssetSelectorExpression.Builder.newInstance() + .whenEquals(Asset.PROPERTY_ID, "test-document") + .build()) + .validity(31536000) + .build(); + contractDefinitionStore.save(contractDefinition); + } + +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java new file mode 100644 index 00000000..055edbdd --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java @@ -0,0 +1,48 @@ +package org.eclipse.edc.sample.extension.policy; + + +import org.eclipse.edc.policy.engine.spi.AtomicConstraintFunction; +import org.eclipse.edc.policy.engine.spi.PolicyContext; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class TimeIntervalFunction implements AtomicConstraintFunction { + + private Monitor monitor; + + public TimeIntervalFunction(Monitor monitor) { + this.monitor = monitor; + } + + + + @Override + public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + Date date; + try { + date = sdf.parse((String) rightValue); + } catch (ParseException e) { + monitor.severe("Failed to parse right value of constraint to date."); + return false; + } + + switch (operator) { + case LT: var isBefore = new Date().before(date); + monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date"); + return isBefore; + case GT: var isAfter = new Date().after(date); + monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date"); + return isAfter; + default: return false; + } + } +} + + diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..1b26f404 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.extension.policy.PolicyFunctionsExtension \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index dbcb5126..9efc69c0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -62,6 +62,11 @@ include("transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") include("util:http-request-logger") + +//policy +include(":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector") +include(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions") + // modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") From e4da7f003128622ab5d3c6629554dadbf8d44e65 Mon Sep 17 00:00:00 2001 From: farhin23 Date: Tue, 21 Mar 2023 15:08:55 +0100 Subject: [PATCH 02/35] for: update checkstyles - create basic policy sample --- .../policy/PolicyFunctionsExtension.java | 23 +++++++++++++++++-- .../policy/TimeIntervalFunction.java | 14 +++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java index a2dca9f6..d18baca3 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + package org.eclipse.edc.sample.extension.policy; import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; @@ -6,7 +20,12 @@ import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; import org.eclipse.edc.policy.engine.spi.PolicyEngine; import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; -import org.eclipse.edc.policy.model.*; +import org.eclipse.edc.policy.model.Action; +import org.eclipse.edc.policy.model.AtomicConstraint; +import org.eclipse.edc.policy.model.LiteralExpression; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.policy.model.Policy; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.asset.AssetSelectorExpression; @@ -23,7 +42,7 @@ public class PolicyFunctionsExtension implements ServiceExtension { private final String policyTimeKey = "POLICY_EVALUATION_TIME"; private final String policyStartDateSetting = "edc.samples.policy-01.constraint.date.start"; private final String policyEndDateSetting = "edc.samples.policy-01.constraint.date.end"; - private final String EDC_ASSET_PATH = "edc.samples.policy-01.asset.path"; + private static final String EDC_ASSET_PATH = "edc.samples.policy-01.asset.path"; @Inject private RuleBindingRegistry ruleBindingRegistry; diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java index 055edbdd..59794255 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + package org.eclipse.edc.sample.extension.policy; From 45de2107cb8ac1b4898410d2a8e9bf49c33a9c89 Mon Sep 17 00:00:00 2001 From: farhin23 Date: Mon, 24 Apr 2023 15:18:12 +0200 Subject: [PATCH 03/35] feat: create basic policy sample - integration tests --- .../build.gradle.kts | 33 +++++ .../BasicPolicySampleConfirmContractTest.java | 59 +++++++++ .../BasicPolicySampleDeclineContractTest.java | 60 +++++++++ .../BasicPolicySampleTestCommon.java | 118 ++++++++++++++++++ .../testFixtures/resources/contractoffer.json | 70 +++++++++++ 5 files changed, 340 insertions(+) create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts new file mode 100644 index 00000000..056dfb10 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * + */ + +plugins { + `java-library` + `java-test-fixtures` +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + testImplementation("$groupId:junit:$edcVersion") + + testFixturesImplementation("$groupId:junit:$edcVersion") + testFixturesImplementation("io.rest-assured:rest-assured:5.3.0") + testFixturesImplementation("org.awaitility", "awaitility", "4.2.0") + testFixturesImplementation("org.assertj", "assertj-core", "3.24.2") + testFixturesImplementation("org.junit.jupiter", "junit-jupiter-api", "5.9.2") + + testCompileOnly(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector")) +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java new file mode 100644 index 00000000..e1274a30 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * + */ + +package org.eclipse.edc.sample.extension; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Map; + +import static org.eclipse.edc.sample.extension.BasicPolicySampleTestCommon.getFileFromRelativePath; + + +/** + * This test class runs test for the scenario when the contract negotiation gets CONFIRMED + * It starts a connector with config.properties, containing a start and end date for which the current time is within the defined interval. + */ +@EndToEndTest +public class BasicPolicySampleConfirmContractTest { + + static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties"; + static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-contract-negotiation/contractoffer.json"; + + @RegisterExtension + static EdcRuntimeExtension connector = new EdcRuntimeExtension( + ":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector", + "connector", + Map.of( + "edc.fs.config", getFileFromRelativePath(CONNECTOR_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() + ) + ); + + final BasicPolicySampleTestCommon testUtils = new BasicPolicySampleTestCommon(); + + /** + * Run all sample steps in one single test. + * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} + * runs before each single test. + */ + @Test + void runSampleSteps() { + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.lookUpContractAgreementConfirmed(); + } + +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java new file mode 100644 index 00000000..cee3a872 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * + */ + +package org.eclipse.edc.sample.extension; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Map; + +import static org.eclipse.edc.sample.extension.BasicPolicySampleTestCommon.getFileFromRelativePath; + + +/** + * This test class runs test for the scenario when the contract negotiation gets DECLINED + * It starts a connector with config.properties, containing a start and end date for which the current time is NOT within the defined interval. + */ +@EndToEndTest +public class BasicPolicySampleDeclineContractTest { + static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties"; + static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json"; + + @RegisterExtension + static EdcRuntimeExtension connector = new EdcRuntimeExtension( + ":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector", + "connector", + Map.of( + // Override 'edc.samples.policy-01.constraint.date.start' & 'edc.samples.policy-01.constraint.date.end' implicitly set via property 'edc.fs.config'. + "edc.samples.policy-01.constraint.date.start", "2022-01-01T00:00:00.000+02:00", + "edc.samples.policy-01.constraint.date.end", "2022-12-31T00:00:00.000+02:00", + "edc.fs.config", getFileFromRelativePath(CONNECTOR_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() + ) + ); + + final BasicPolicySampleTestCommon testUtils = new BasicPolicySampleTestCommon(); + + /** + * Run all sample steps in one single test. + * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} + * runs before each single test. + */ + @Test + void runSampleSteps() { + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.lookUpContractAgreementDeclined(); + } +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java new file mode 100644 index 00000000..d0bb0b41 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * + */ + +package org.eclipse.edc.sample.extension; + + +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.time.Duration; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +/** + * Encapsulates common settings, test steps, and helper methods for the test for {@code :policy:policy-01-contract-negotiation} sample. + */ +public class BasicPolicySampleTestCommon { + + //region constant test settings + static final String INITIATE_CONTRACT_NEGOTIATION_URI = "http://localhost:8182/api/v1/management/contractnegotiations"; + static final String LOOK_UP_CONTRACT_AGREEMENT_URI = "http://localhost:8182/api/v1/management/contractnegotiations/{id}"; + static final String API_KEY_HEADER_KEY = "X-Api-Key"; + static final String API_KEY_HEADER_VALUE = "password"; + //endregion + + //region changeable test settings + Duration timeout = Duration.ofSeconds(30); + Duration pollInterval = Duration.ofMillis(500); + //endregion + + String contractNegotiationId; + + /** + * Creates a new {@link BasicPolicySampleTestCommon} instance. + */ + public BasicPolicySampleTestCommon() { + } + + /** + * Resolves a {@link File} instance from a relative path. + */ + @NotNull + public static File getFileFromRelativePath(String relativePath) { + return new File(TestUtils.findBuildRoot(), relativePath); + } + + /** + * Assert that a POST request to initiate a contract negotiation is successful. + * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/api/v1/management/contractnegotiations"} + */ + void initiateContractNegotiation(String contractOfferFilePath) { + contractNegotiationId = RestAssured + .given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) + .when() + .post(INITIATE_CONTRACT_NEGOTIATION_URI) + .then() + .statusCode(HttpStatus.SC_OK) + .body("id", not(emptyString())) + .extract() + .jsonPath() + .get("id"); + } + + /** + * Assert that a GET request to look up a contract agreement is successful and the {@code state} is {@code 'CONFIRMED'}. + * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/api/v1/management/contractnegotiations/{UUID}"} + */ + void lookUpContractAgreementConfirmed() { + // Wait for transfer to be completed. + await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> RestAssured + .given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .when() + .get(LOOK_UP_CONTRACT_AGREEMENT_URI, contractNegotiationId) + .then() + .statusCode(HttpStatus.SC_OK) + .body("state", equalTo("CONFIRMED")) + ); + } + + /** + * Assert that a GET request to look up a contract agreement is successful and the {@code state} is {@code 'DECLINED'}. + * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/api/v1/management/contractnegotiations/{UUID}"} + */ + void lookUpContractAgreementDeclined() { + // Wait for transfer to be completed. + await().atMost(Duration.ofSeconds(120)).pollInterval(pollInterval).untilAsserted(() -> RestAssured + .given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .when() + .get(LOOK_UP_CONTRACT_AGREEMENT_URI, contractNegotiationId) + .then() + .statusCode(HttpStatus.SC_OK) + .body("state", equalTo("DECLINED")) + ); + } +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json new file mode 100644 index 00000000..faa2327a --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json @@ -0,0 +1,70 @@ +{ + "connectorId": "provider", + "connectorAddress": "http://localhost:8282/api/v1/ids/data", + "protocol": "ids-multipart", + "offer": { + "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "assetId": "test-document", + "policy": { + "uid": "956e172f-2de1-4501-8881-057a57fd0e69", + "permissions": [ + { + "edctype": "dataspaceconnector:permission", + "uid": null, + "target": "test-document", + "action": { + "type": "USE", + "includedIn": null, + "constraint": null + }, + "assignee": null, + "assigner": null, + "constraints": [ + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2022-01-01T00:00:00.000+02:00" + }, + "operator": "GT" + }, + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2022-12-31T00:00:00.000+02:00" + }, + "operator": "LT" + } + ], + "duties": [] + } + ], + "prohibitions": [], + "obligations": [], + "extensibleProperties": {}, + "inheritsFrom": null, + "assigner": null, + "assignee": null, + "target": null, + "@type": { + "@policytype": "set" + } + }, + "asset": { + "properties": { + "ids:byteSize": null, + "asset:prop:id": "test-document", + "ids:fileName": null + } + } + } +} \ No newline at end of file From dd495bb51bdf1530abd14857f89537635b571f70 Mon Sep 17 00:00:00 2001 From: farhin23 Date: Wed, 3 May 2023 14:48:02 +0200 Subject: [PATCH 04/35] for: update checkstyles - copyright --- .../build.gradle.kts | 4 ++-- .../extension/BasicPolicySampleConfirmContractTest.java | 4 ++-- .../extension/BasicPolicySampleDeclineContractTest.java | 4 ++-- .../edc/sample/extension/BasicPolicySampleTestCommon.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts index 056dfb10..6db43af5 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Microsoft Corporation + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -8,7 +8,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: - * Microsoft Corporation - initial test implementation for sample + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation * */ diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java index e1274a30..aedd3eff 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Microsoft Corporation + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -8,7 +8,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: - * Microsoft Corporation - initial test implementation for sample + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation * */ diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java index cee3a872..1501c239 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Microsoft Corporation + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -8,7 +8,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: - * Microsoft Corporation - initial test implementation for sample + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation * */ diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java index d0bb0b41..8fcb92bc 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Microsoft Corporation + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at @@ -8,7 +8,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: - * Microsoft Corporation - initial test implementation for sample + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation * */ From c549da13a57475746b62a3033918919ac5f59e03 Mon Sep 17 00:00:00 2001 From: farhin23 Date: Wed, 3 May 2023 14:55:45 +0200 Subject: [PATCH 05/35] for: basic policy sample - add commented out modules - update dependencies in libs.versions.toml --- .../build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts index 6db43af5..a497a186 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts @@ -24,10 +24,10 @@ dependencies { testImplementation("$groupId:junit:$edcVersion") testFixturesImplementation("$groupId:junit:$edcVersion") - testFixturesImplementation("io.rest-assured:rest-assured:5.3.0") - testFixturesImplementation("org.awaitility", "awaitility", "4.2.0") - testFixturesImplementation("org.assertj", "assertj-core", "3.24.2") - testFixturesImplementation("org.junit.jupiter", "junit-jupiter-api", "5.9.2") + testFixturesImplementation(libs.restAssured) + testFixturesImplementation(libs.awaitility) + testFixturesImplementation(libs.assertj) + testFixturesImplementation(libs.junit.jupiter.api) testCompileOnly(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector")) } \ No newline at end of file From cff651d3df5f09e088b3e6faa2da31a1a6a3c089 Mon Sep 17 00:00:00 2001 From: Ronja Quensel Date: Mon, 11 Sep 2023 14:07:28 +0200 Subject: [PATCH 06/35] chore: update policy 1 README --- README.md | 8 + policy/README.md | 12 + .../policy-01-contract-negotiation/README.md | 147 --------- .../contractoffer.json | 70 ---- policy/policy-01-policy-enforcement/README.md | 306 ++++++++++++++++++ .../contractoffer.json | 24 ++ .../build.gradle.kts | 7 +- .../build.gradle.kts | 2 +- .../BasicPolicySampleConfirmContractTest.java | 0 .../BasicPolicySampleDeclineContractTest.java | 0 .../BasicPolicySampleTestCommon.java | 0 .../testFixtures/resources/contractoffer.json | 0 .../build.gradle.kts | 29 ++ .../config.properties | 0 .../policy-functions}/build.gradle.kts | 0 .../policy/PolicyFunctionsExtension.java | 0 .../policy/TimeIntervalFunction.java | 0 ...rg.eclipse.edc.spi.system.ServiceExtension | 0 18 files changed, 381 insertions(+), 224 deletions(-) create mode 100644 policy/README.md delete mode 100644 policy/policy-01-contract-negotiation/README.md delete mode 100644 policy/policy-01-contract-negotiation/contractoffer.json create mode 100644 policy/policy-01-policy-enforcement/README.md create mode 100644 policy/policy-01-policy-enforcement/contractoffer.json rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-connector => policy-01-policy-enforcement/policy-enforcement-consumer}/build.gradle.kts (72%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-integration-tests => policy-01-policy-enforcement/policy-enforcement-integration-tests}/build.gradle.kts (89%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-integration-tests => policy-01-policy-enforcement/policy-enforcement-integration-tests}/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-integration-tests => policy-01-policy-enforcement/policy-enforcement-integration-tests}/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-integration-tests => policy-01-policy-enforcement/policy-enforcement-integration-tests}/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-integration-tests => policy-01-policy-enforcement/policy-enforcement-integration-tests}/src/testFixtures/resources/contractoffer.json (100%) create mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-connector => policy-01-policy-enforcement/policy-enforcement-provider}/config.properties (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-policy-functions => policy-01-policy-enforcement/policy-functions}/build.gradle.kts (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-policy-functions => policy-01-policy-enforcement/policy-functions}/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-policy-functions => policy-01-policy-enforcement/policy-functions}/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java (100%) rename policy/{policy-01-contract-negotiation/policy-contract-negotiation-policy-functions => policy-01-policy-enforcement/policy-functions}/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension (100%) diff --git a/README.md b/README.md index f671360a..ee72c873 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,14 @@ EDC and run different transfer scenarios. Click the link above to learn about th All transfer samples are located in the `transfer` directory. +### [Policy](./policy/README.md) + +These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what +configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide +custom code for their enforcement. + +All policy samples are located in the `policy` directory. + ## Contributing See [how to contribute](CONTRIBUTING.md). diff --git a/policy/README.md b/policy/README.md new file mode 100644 index 00000000..d9853f7c --- /dev/null +++ b/policy/README.md @@ -0,0 +1,12 @@ +# Policy samples + +The samples in this scope revolve around the topic of policies and policy evaluation/enforcement. + +> Before starting with these samples, be sure to check out the [basic samples](../basic/README.md)! + +## Samples + +### [Policy sample 01](./policy-01-policy-enforcement/README.md): Policy enforcement + +This sample will teach you the very basics of policy enforcement. It shows the necessary configurations to enable +policy evaluation for certain rules as well as an example on how to enforce a policy with a constraint. diff --git a/policy/policy-01-contract-negotiation/README.md b/policy/policy-01-contract-negotiation/README.md deleted file mode 100644 index 5778a1da..00000000 --- a/policy/policy-01-contract-negotiation/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Create a policy function - -This sample shows how a custom function for policy evaluation can be implemented and registered. For simplicity, -we will use just one connector, which initiates a contract negotiation with itself. The sample contains two modules: - -- `policy-contract-negotiation-connector`: contains the connector configuration -- `policy-contract-negotiation-policy-functions`: provides the function for policy evaluation - -## Creating and registering the function - -The [`TimeIntervalFunction`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java) -implements the `AtomicConstraintFunction` interface, which contains a single method for evaluating a constraint. -In that method, the `operator` and `right value` of the constraint can be used for evaluation. In this example, we -want to check whether the evaluation happens within a specified time interval. So depending on whether the operator is -`GT` or `LT`, we check if the current date is before or after the date specified in the `right value`. - -```java -@Override -public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { - var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - - Date date; - try { - date = sdf.parse((String) rightValue); - } catch (ParseException e) { - monitor.severe("Failed to parse right value of constraint to date."); - return false; - } - - switch (operator) { - case LT: var isBefore = new Date().before(date); - monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date."); - return isBefore; - case GT: var isAfter = new Date().after(date); - monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date."); - return isAfter; - default: return false; - } -} -``` - -Next, we have to register our function with the `PolicyEngine` and bind the desired action as well as the key used to -register our function to the desired scopes using the `RuleBindingRegistry`. This is done in the -[`PolicyFunctionsExtension`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java): - -```java -private final String policyTimeKey = "POLICY_EVALUATION_TIME"; - -//... - -@Override -public void initialize(ServiceExtensionContext context) { - //... - - ruleBindingRegistry.bind("USE", ALL_SCOPES); - ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); - policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); - - //... -} -``` - -## Defining the policy - -Next to registering the function and creating the binding, the `PolicyFunctionsExtension` also creates an asset, -a policy and a contract definition linking the former two. The asset is the same one that is used in -sample [`transfer-01-file-transfer`](transfer/transfer-01-file-transfer/README.md), but this time a different policy is created. - -We start by creating the constraints, where we define that the policy evaluation time must be within a certain -time interval. We read the start and end dates for the interval from the settings. Then, we create two constraints: -one specifying that the evaluation time should be after our defined start date, and one specifying that the evaluation -time should be before our defined end date. **We need to set the constraints' left operands to the same key -that we previously used to register our function. Otherwise, the function will not be used to evaluate these -constraints.** - -```java -var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); -var notBeforeConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.GT) - .rightExpression(new LiteralExpression(startDate)) - .build(); - -var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T23:59:00.000+02:00"); -var notAfterConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.LT) - .rightExpression(new LiteralExpression(endDate)) - .build(); -``` - -Then, we create a `Permission` with action type `USE` and the two constraints. We use this permission to create and -store a policy. And last, we create the `ContractDefinition`. For the access policy, we use the same use-policy that is -used in sample `transfer:transfer-01-file-transfer`, and for the contract policy, we use the previously created policy with the time -interval restriction. We set the `AssetSelectorExpression` so that the contract definition is valid for our asset. - -## How to run the sample - -To see the policy enforcement working, this sample should be run twice with different configurations. - -### Configuration - -Choose one of the following configurations. Depending on which configuration you choose, the contract negotiation -will either be confirmed or declined. - -#### 1. Policy fulfilled - -Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is within -the defined interval. - -```properties -edc.samples.uc.constraint.date.start=2023-01-01T00:00:00.000+02:00 -edc.samples.uc.constraint.date.end=2023-12-31T23:59:00.000+02:00 -``` - -#### 2. Policy not fulfilled - -Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is -**not** within the defined interval. - -```properties -edc.samples.policy-01.constraint.date.start=2022-01-01T00:00:00.000+02:00 -edc.samples.policy-01.constraint.date.end=2022-12-31T23:59:00.000+02:00 -``` - -### Run the sample - -First, build and run the connector for this sample: - -```shell -./gradlew policy:policy-01-contract-negotiation:policy-contract-negotiation-connector:build -java -Dedc.fs.config=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar -#for windows: -java -D"edc.fs.config"=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar -``` - -Next, initiate a contract negotiation. The request body is prepared in [`contractoffer.json`](policy/policy-01-contract-negotiation/contractoffer.json). -In that file, replace the dates for the constraints with the dates you used in the `config.properties` file and then -run: - -```shell -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @samples/uc-workshop/contractoffer.json "http://localhost:8182/api/v1/management/contractnegotiations" -``` - -Depending on the configuration you chose, you can see the contract negotiation either being confirmed or declined -in the connector logs. The logs also contain messages logged by the `TimeIntervalFunction`, which tell whether the -current date is before or after the start and end dates defined in the policy. diff --git a/policy/policy-01-contract-negotiation/contractoffer.json b/policy/policy-01-contract-negotiation/contractoffer.json deleted file mode 100644 index 9af961ad..00000000 --- a/policy/policy-01-contract-negotiation/contractoffer.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "connectorId": "provider", - "connectorAddress": "http://localhost:8282/api/v1/ids/data", - "protocol": "ids-multipart", - "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", - "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [ - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2023-01-01T00:00:00.000+02:00" - }, - "operator": "GT" - }, - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2023-12-31T00:00:00.000+02:00" - }, - "operator": "LT" - } - ], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } - } - } -} \ No newline at end of file diff --git a/policy/policy-01-policy-enforcement/README.md b/policy/policy-01-policy-enforcement/README.md new file mode 100644 index 00000000..f45c571f --- /dev/null +++ b/policy/policy-01-policy-enforcement/README.md @@ -0,0 +1,306 @@ +# Policy enforcement + +In this sample we'll learn how to enable policy enforcement. As the EDC are a framework, they do not provide any +evaluation out-of-the-box, but instead provide an evaluation system that can be easily configured to suit custom needs. +We'll perform the necessary configurations and implement and register a function for evaluating a policy. + +We will set up two connectors, a provider and a consumer, and let the provider offer an asset with a policy that +imposes a location restriction. So depending on the consumer's location, the consumer will be able to negotiate a +contract for requesting the asset or not. The sample consists of multiple modules: + +* `policy-functions`: creates the provider's offer and provides the function for policy enforcement +* `[policy-enforcement-provider|consumer]`: contains the build and config files for the respective connector + +## Creating the policy functions extension + +In this extension, we'll define the policy and create the provider's offer. In addition, we'll implement and register +a function to evaluate the defined policy. + +### Creating the provider offer + +In order for the provider to offer any data, we need to create 3 things: an `Asset` (= what data should be offered), +a `PolicyDefinition` (= under which conditions should data be offered), and a `ContractDefinition`, that links the +`Asset` and `PolicyDefinition`. + +We'll start with creating the `PolicyDefinition`, which contains a `Policy` and an ID. Each `Policy` needs to contain +at least one rule describing which actions are allowed, disallowed or required to perform. Each rule can optionally +contain a set of constraints that further refine the actions. For more information on the policy model take a look at +the [documentation](https://github.com/eclipse-edc/Connector/blob/main/docs/developer/architecture/usage-control/policies.md) +or the [policy section in the developer handbook](https://github.com/eclipse-edc/docs/blob/main/developer/handbook.md#policies). + +For our example, we create a `Permission` with action type `USE`, as we want to allow the usage of our offered data. +But we only want to allow the usage under the condition that the requesting participant is in a certain location, +therefore we define a constraint. In that constraint we state that the participant's location has to be equal to `eu`. +We then add the constraint to our permission and the permission to our policy. Lastly, we give an ID to our policy to +create the `PolicyDefinition`: + +```java +var locationConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(LOCATION_CONSTRAINT_KEY)) + .operator(Operator.EQ) + .rightExpression(new LiteralExpression("eu")) + .build(); +var permission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance() + .type("USE") + .build()) + .constraint(locationConstraint) + .build(); +var policyDefinition = PolicyDefinition.Builder.newInstance() + .id(LOCATION_POLICY_ID) + .policy(Policy.Builder.newInstance() + .type(PolicyType.SET) + .permission(permission) + .build()) + .build(); + +policyDefinitionStore.create(policyDefinition); +``` + +Then, we create an `Asset` with a `DataAddress` of type *test*. This asset will **not** work for a data transfer, +as *test* is not an actual transfer type. But, as we're not going to transfer any data in this sample, this is +sufficient for our example. The last thing we create is a `ContractDefinition` that references the previously created +policy definition and asset. With this, the provider offers the asset under the condition that the +requesting participant is located in the EU. + +### Creating rule bindings + +The provider now offers an asset with a policy that imposes a constraint, but if we were to run the sample now, +we would not see any policy evaluation happening. This is because the EDC do not regard any rules or constraints +for evaluation unless we configure it. The EDC use the concept of *policy scopes* to define which rules and constraints +should be evaluated in certain runtime contexts, as some rules or constraints may only make sense in some contexts, +but not in others. A simple example is a rule that states *data must be anonymized*. Evaluating this during the +contract negotiation would not make much sense, as at this point in time no data is being exchanged yet and therefore +nothing can be anonymized. So we need to define which rules and constraints should be evaluated in which scopes. +This is done by creating *rule bindings* at the `RuleBindingRegistry`. For our example, we create the following rule +bindings: + +```java +ruleBindingRegistry.bind("USE", ALL_SCOPES); +ruleBindingRegistry.bind(LOCATION_CONSTRAINT_KEY, NEGOTIATION_SCOPE); +``` + +When creating a rule binding, we can bind an action type or constraint to either all scopes or just a specific one. +Here, we bind the action type `USE` to all scopes, so that rules with this action type are always evaluated. For the +location constraint we choose the negotiation scope, meaning it will only be evaluated during the contract negotiation. +Information on available scopes can be found [here](https://github.com/eclipse-edc/Connector/blob/main/docs/developer/policy-engine.md). + +### Implementing the function for evaluation + +With the rule bindings in place, the provider will now try to evaluate our policy including the constraint during a +contract negotiation, but it does not yet know *how* to evaluate this constraint. For this, we need to implement a +function, for which the EDC offer two interfaces: `AtomicConstraintFunction` and `RuleFunction`. The former is meant +for evaluating a single constraint of a rule, while is latter is meant for evaluating a complete rule node (including +constraints as well as duties that may be associated with a permission). For our example, we choose to implement an +`AtomicConstraintFunction`, as we want to evaluate our location constraint: + +```java +public class LocationConstraintFunction implements AtomicConstraintFunction { + + //... + + @Override + public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + var region = context.getParticipantAgent().getClaims().get("region"); + + monitor.info(format("Evaluating constraint: location {} {}", operator, rightValue.toString())); + + return switch (operator) { + case EQ -> Objects.equals(region, rightValue); + case NEQ -> !Objects.equals(region, rightValue); + case IN -> ((Collection) rightValue).contains(region); + default -> false; + }; + } +} +``` + +When implementing either of the function interfaces, we have to override the `evaluate` method. For the +`AtomicConstraintFunction` we get the constraint's operator and right value as well as the containing rule node and +a `PolicyContext` as parameters. Using these, we have to determine whether the constraint is fulfilled. Since we want +to check the requesting participant's location, we need to access information about the participant. This is supplied +through the `PolicyContext`. We get the participant's claim with key *region* to obtain information about the +participant's location. We can then compare the location to the expected value depending on the operator used. The +function should return true, if the constraint is fulfilled, and false otherwise. + +**Note**: we can use the *region* claim here because our connectors use the `iam-mock` extension, which always adds +a claim with this exact name to all tokens. Depending on the identity provider used, different claims may be present, +or the same claim may have a different name. + +### Registering the function with the policy engine + +After creating our function for evaluation, the last thing we need to do is register this function at the +`PolicyEngine`, so that it is available for evaluation: + +```java +policyEngine.registerFunction(NEGOTIATION_SCOPE, Permission.class, LOCATION_CONSTRAINT_KEY, new LocationConstraintFunction(monitor)); +``` + +When registering the function, we again have to specify a scope. This allows for evaluating the same rule or +constraint differently in different runtime contexts. Since we bound our constraint to the negotiation scope, we also +register our function for that scope. Next, we need to specify the type of rule our function should be used for. This +is important, as the same constraint may have different implications as part of a permission, prohibition or duty. +When registering an `AtomicConstraintFunction`, we also have to specify a key that the function is associated with. +This has to resolve to exactly the constraint's left operand, so that the correct function for evaluation of a +constraint can be chosen depending on its left operand. So we set the key to the same value we used as our constraint's +left operand. And lastly, we hand over an instance of our function. + +Now, during a contract negotiation, our provider will evaluate our constraint by calling our function's `evaluate` +method. + +## Configuring the connectors + +Next, let's configure the two connectors. For each connector we need a build file and a configuration file. + +### Build files + +In the build file, we define the following dependencies for both connectors: + +* `libs.edc.control.plane.core`: the core module for the control-plane +* `libs.edc.configuration.filesystem`: enables configuration via a properties file +* `libs.edc.dsp`: enables connector-to-connector communication via the Dataspace Protocol +* `libs.edc.iam.mock`: mocks an identity provider + +**Note**: we do not include any `data-plane` modules, as we are not going to transfer any data in this sample. To +be able to actually transfer data, additional dependencies are required. More information can be found in the +[documentation](https://github.com/eclipse-edc/Connector/blob/main/docs/developer/build-your-own-connector.md) and in +the [transfer samples](../../transfer/README.md). + +#### Provider + +For the provider, we also add a dependency on our previously created `policy-functions` extension, so that it offers +our asset with the location restricted policy and is able to enforce the latter. + +#### Consumer + +In addition, we add the following dependency on the consumer side, as we will use the management API to initiate a +contract negotiation between provider and consumer: + +* `libs.edc.management.api`: provides the API for interacting with the control-plane + +### Configuration files + +We create the `config.properties` files for both provider and consumer and first define their API bindings. We then +define the DSP callback addresses, which are required for callback during the contract negotiation, as well as their +participant IDs. + +#### Consumer + +For the consumer we also add the following property: + +```properties +edc.mock.region=us +``` + +This defines the value for the consumer's region claim issued by the mock identity provider, which we use for evaluating +the consumer's location. + +## Running the sample + +Now, let's run the sample step by step. + +### 1. Run connectors + +First, we need to build and start both our connectors. Execute the following commands from the project root in two +separate terminal windows (one per connector): + +Provider: +```bash +./gradlew policy:policy-01-policy-enforcement:policy-enforcement-provider:build +java -Dedc.fs.config=policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties -jar policy/policy-01-policy-enforcement/policy-enforcement-provider/build/libs/provider.jar +``` + +Consumer: +```bash +./gradlew policy:policy-01-policy-enforcement:policy-enforcement-consumer:build +java -Dedc.fs.config=policy/policy-01-policy-enforcement/policy-enforcement-consumer/config.properties -jar policy/policy-01-policy-enforcement/policy-enforcement-consumer/build/libs/consumer.jar +``` + +### 2. Start a contract negotiation + +To start the contract negotiation between provider and consumer, we'll use an endpoint of the consumer's management API. +In the request body for this request, we need to provide information about which connector we want to negotiate with, +which protocol to use and which offer we want to negotiate. The request body is prepared in +[contractoffer.json](./contractoffer.json). To start the negotiation, run the following command: + +```bash +curl -X POST -H "Content-Type: application/json" -d @policy/policy-01-policy-enforcement/contractoffer.json "http://localhost:9192/api/management/v2/contractnegotiations" +``` + +You'll get back a UUID. This is the ID of the contract negotiation process which is being asynchronously executed +in the background. + +### 3. Get the contract negotiation state + +Using the ID received in the previous step, we can now view the state of the negotiation by calling another endpoint +of the consumer's data management API: + +```bash +curl -X GET "http://localhost:9192/api/management/v2/contractnegotiations/" +``` + +In the response we'll get a description of the negotiation, similar to the following: + +```json +{ + ... + "edc:contractAgreementId": null, + "edc:state": "DECLINED", + ... +} +``` + +We can see that the negotiation has been declined, and we did not receive a contract agreement. If we now take a look +at the provider's logs, we'll see the following lines: + +```bash +[TODO: copy provider logs showing the failed policy evaluation] +``` + +The consumer was not able to get a contract agreement, because it does not fulfil the location-restricted policy. This +means we have successfully implemented and configured the policy evaluation on provider side. Building up on this +example, you can now tackle more complex policies, by e.g. defining and combining different constraints and creating +the respective functions for evaluation. + +## Sample variations + +You can play around with this sample a bit and run it in different variations, yielding different outcomes. Some +possible variations are described in the following. + +### Set consumer region to `eu` + +Our policy requires the consumer to be in the EU. Change the property `edc.mock.region` in the consumer's +`config.properties` to the value `eu` and run the sample again. This time, the negotiation will reach the state +`FINALIZED` and reference a contract agreement, as our consumer now fulfils the policy. + +```properties +edc.mock.region=eu +``` + +### Remove binding of constraint + +In our `PolicyFunctionsExtension`, we've created a rule binding so that our constraint would be evaluated during the +contract negotiation. Remove this binding and run the sample again (while leaving the consumer's property +`edc.mock-region` with value `us`!). The negotiation will be confirmed and reference a contract agreement, even though +our consumer is not in the correct location. This happens, as without the binding of the constraint, the provider +will not regard it during evaluation. + +```java +ruleBindingRegistry.bind("USE", ALL_SCOPES); +//ruleBindingRegistry.bind(LOCATION_CONSTRAINT_KEY, NEGOTIATION_SCOPE); +``` + +### Remove binding of action type + +In our `PolicyFunctionsExtension`, we've created rule bindings for our permission's action type as well as the +constraint. In the previous variation, we've removed the binding for the constraint. For this variation, we want to +leave the binding for the constraint in place, and instead remove the binding for the action type. Run the sample again +(while leaving the consumer's property `edc.mock-region` with value `us`!) and you will see the negotiation being +confirmed. Even though the constraint is bound to be evaluated and the consumer does not fulfil it, the constraint +is not evaluated and our function never called. This happens because there is no rule binding for the permission +containing the constraint, and thus the whole permission node is disregarded during evaluation. + +```java +//ruleBindingRegistry.bind("USE", ALL_SCOPES); +ruleBindingRegistry.bind(LOCATION_CONSTRAINT_KEY, NEGOTIATION_SCOPE); +``` diff --git a/policy/policy-01-policy-enforcement/contractoffer.json b/policy/policy-01-policy-enforcement/contractoffer.json new file mode 100644 index 00000000..6fcf1a0c --- /dev/null +++ b/policy/policy-01-policy-enforcement/contractoffer.json @@ -0,0 +1,24 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "NegotiationInitiateRequestDto", + "connectorId": "provider", + "consumerId": "consumer", + "providerId": "provider", + "connectorAddress": "http://localhost:8282/api/protocol", + "protocol": "dataspace-protocol-http", + "offer": { + "offerId": "", + "assetId": "", + "policy": { + "@id": "", + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "" + } + } +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts similarity index 72% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts rename to policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts index a60d9218..cad20478 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts @@ -10,17 +10,12 @@ val edcVersion: String by project dependencies { implementation("$groupId:control-plane-core:$edcVersion") - implementation("$groupId:api-observability:$edcVersion") - implementation("$groupId:configuration-filesystem:$edcVersion") implementation("$groupId:iam-mock:$edcVersion") - implementation("$groupId:auth-tokenbased:$edcVersion") implementation("$groupId:management-api:$edcVersion") implementation("$groupId:ids:$edcVersion") - - implementation(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions")) } application { @@ -30,5 +25,5 @@ application { tasks.withType { exclude("**/pom.properties", "**/pom.xm") mergeServiceFiles() - archiveFileName.set("connector.jar") + archiveFileName.set("consumer.jar") } diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts similarity index 89% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts rename to policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts index a497a186..ee794719 100644 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts @@ -29,5 +29,5 @@ dependencies { testFixturesImplementation(libs.assertj) testFixturesImplementation(libs.junit.jupiter.api) - testCompileOnly(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector")) + testCompileOnly(project(":policy:policy-01-policy-enforcement:policy-enforcement-provider")) } \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java rename to policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java rename to policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java rename to policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json rename to policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts new file mode 100644 index 00000000..d729fe0b --- /dev/null +++ b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts @@ -0,0 +1,29 @@ +plugins { + `java-library` + id("application") + id("com.github.johnrengelman.shadow") version "7.1.2" +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + implementation("$groupId:control-plane-core:$edcVersion") + + implementation("$groupId:configuration-filesystem:$edcVersion") + implementation("$groupId:iam-mock:$edcVersion") + + implementation("$groupId:ids:$edcVersion") + + implementation(project(":policy:policy-01-policy-enforcement:policy-functions")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + exclude("**/pom.properties", "**/pom.xm") + mergeServiceFiles() + archiveFileName.set("provider.jar") +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties b/policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties rename to policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts rename to policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java rename to policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java rename to policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-01-policy-enforcement/policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension similarity index 100% rename from policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension rename to policy/policy-01-policy-enforcement/policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension From d4916117627807beeb26bfbca5f043274f65fedc Mon Sep 17 00:00:00 2001 From: Mohammad Majadly Date: Fri, 15 Sep 2023 16:20:43 +0200 Subject: [PATCH 07/35] chore: update build and settings of gradle --- .../policy-enforcement-consumer/build.gradle.kts | 15 ++++++--------- .../build.gradle.kts | 6 ++---- .../policy-enforcement-provider/build.gradle.kts | 12 +++++------- .../policy-functions/build.gradle.kts | 7 ++----- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts index cad20478..40513d2e 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts @@ -1,21 +1,18 @@ plugins { `java-library` id("application") - id("com.github.johnrengelman.shadow") version "7.1.2" + alias(libs.plugins.shadow) } -val groupId: String by project -val edcVersion: String by project - dependencies { - implementation("$groupId:control-plane-core:$edcVersion") + implementation(libs.edc.control.plane.core) - implementation("$groupId:configuration-filesystem:$edcVersion") - implementation("$groupId:iam-mock:$edcVersion") + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) - implementation("$groupId:management-api:$edcVersion") + implementation(libs.edc.management.api) - implementation("$groupId:ids:$edcVersion") + implementation(libs.edc.ids) } application { diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts index ee794719..ca5a0883 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts @@ -17,13 +17,11 @@ plugins { `java-test-fixtures` } -val groupId: String by project -val edcVersion: String by project dependencies { - testImplementation("$groupId:junit:$edcVersion") + testImplementation(libs.edc.junit) - testFixturesImplementation("$groupId:junit:$edcVersion") + testFixturesImplementation(libs.edc.junit) testFixturesImplementation(libs.restAssured) testFixturesImplementation(libs.awaitility) testFixturesImplementation(libs.assertj) diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts index d729fe0b..9d6023b2 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts @@ -1,19 +1,17 @@ plugins { `java-library` id("application") - id("com.github.johnrengelman.shadow") version "7.1.2" + alias(libs.plugins.shadow) } -val groupId: String by project -val edcVersion: String by project dependencies { - implementation("$groupId:control-plane-core:$edcVersion") + implementation(libs.edc.control.plane.core) - implementation("$groupId:configuration-filesystem:$edcVersion") - implementation("$groupId:iam-mock:$edcVersion") + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) - implementation("$groupId:ids:$edcVersion") + implementation(libs.edc.dsp) implementation(project(":policy:policy-01-policy-enforcement:policy-functions")) } diff --git a/policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts index df01ec91..fe635374 100644 --- a/policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-functions/build.gradle.kts @@ -3,12 +3,9 @@ plugins { id("application") } -val groupId: String by project -val edcVersion: String by project - dependencies { - api("$groupId:data-plane-spi:$edcVersion") + api(libs.edc.data.plane.spi) - implementation("$groupId:control-plane-core:$edcVersion") + implementation(libs.edc.control.plane.core) } From db3387cf135c827dafaf443f7e227711c81764f0 Mon Sep 17 00:00:00 2001 From: Mohammad Majadly Date: Thu, 5 Oct 2023 16:17:15 +0200 Subject: [PATCH 08/35] chore: update build and settings of gradle --- settings.gradle.kts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 9efc69c0..a8884dbb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -64,8 +64,10 @@ include("util:http-request-logger") //policy -include(":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector") -include(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions") +include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") +include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") +include(":policy:policy-01-policy-enforcement:policy-functions") +include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") // modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") From 2ea27686b1938b272f63f12217eff264206e4a57 Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Fri, 6 Oct 2023 08:43:31 +0200 Subject: [PATCH 09/35] fix: #86 sample 04 terminating on startup (#128) * fix: working sample 04 * test: added test for docker compose sample * style: cosmetic changes * test: temporarily disable progress checking of transfer * docs: added explanation for commented line * test: download OpenTelemetry Jar before test execution * gradle: download OpenTelemetry before build step * removing container name from prometheus as it's not supported by testcontainers * fixed typo in libs.versions.toml * copy opentelemetry jar from gradle cache * fixed default sample tests for sample 04 * fixed build process * asserting Jaeger state OK in telemetry test * fix: fixing wrong testcontainer reference in .toml file * fix: running gradle build before test * fix: removing gradle build before test for github action --------- Co-authored-by: Matthias De Geyter --- .github/workflows/verify.yaml | 3 +- gradle/libs.versions.toml | 10 ++- system-tests/build.gradle.kts | 3 + .../FileTransferSampleTestCommon.java | 17 +++- .../transfer/Transfer04openTelemetryTest.java | 84 +++++++++++++++++++ transfer/transfer-04-open-telemetry/README.md | 7 +- .../contractoffer.json | 47 ++++------- .../docker-compose.yaml | 17 ++-- .../filetransfer.json | 21 ++--- .../transfer-04-open-telemetry/input-file.txt | 1 - .../open-telemetry-consumer/build.gradle.kts | 38 ++++++++- .../open-telemetry-provider/build.gradle.kts | 31 +++++++ 12 files changed, 212 insertions(+), 67 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java delete mode 100644 transfer/transfer-04-open-telemetry/input-file.txt diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b6ee42e3..27141139 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,7 +46,8 @@ jobs: - name: End to End Integration Tests uses: ./.github/actions/run-tests with: - command: ./gradlew test -DincludeTags="EndToEndTest" + command: + - ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 391544d0..d467c6a0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,9 +12,8 @@ okhttp-mockwebserver = "5.0.0-alpha.11" openTelemetry = "1.18.0" restAssured = "5.3.2" rsApi = "3.1.0" +testcontainers = "1.19.1" kafkaClients = "3.6.0" -testContainers = "1.19.1" - [libraries] assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } @@ -69,9 +68,12 @@ junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "jun okhttp-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp-mockwebserver" } opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version.ref = "openTelemetry" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } -testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testContainers" } -testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testContainers" } +testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } +testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } +opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.30.0" [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index c5276dfa..8d033245 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.okhttp.mockwebserver) testImplementation(libs.restAssured) + testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.junit) testImplementation(libs.testcontainers.kafka) testImplementation(libs.kafka.clients) @@ -36,6 +37,8 @@ dependencies { testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider")) testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer")) testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index be7d0474..78e100bb 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon { final String sampleAssetFilePath; final File sampleAssetFile; final File destinationFile; - Duration timeout = Duration.ofSeconds(30); + Duration timeout = Duration.ofSeconds(15); Duration pollInterval = Duration.ofMillis(500); String contractNegotiationId; @@ -133,10 +133,15 @@ void assertTransferProcessStatusConsumerSide(String transferProcessId) { * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"} */ void initiateContractNegotiation() { + initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + } + + + void initiateContractNegotiation(String contractOfferFilePath) { contractNegotiationId = given() .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), CONTRACT_OFFER_FILE_PATH)) + .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) .when() .post(MANAGEMENT_API_URL + "/v2/contractnegotiations") .then() @@ -181,8 +186,8 @@ void lookUpContractAgreementId() { * * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}. */ - String requestTransferFile() throws IOException { - var transferJsonFile = getFileFromRelativePath(TRANSFER_FILE_PATH); + String requestTransferFile(String transferFilePath) throws IOException { + var transferJsonFile = getFileFromRelativePath(transferFilePath); var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId); var jsonPath = given() @@ -205,6 +210,10 @@ String requestTransferFile() throws IOException { return transferProcessId; } + String requestTransferFile() throws IOException { + return requestTransferFile(TRANSFER_FILE_PATH); + } + /** * Reads a transfer request file with changed value for contract agreement ID and file destination path. * diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java new file mode 100644 index 00000000..eb3d7029 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.junit.ClassRule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +@EndToEndTest +@Testcontainers +public class Transfer04openTelemetryTest { + + private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; + private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; + private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; + private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; + private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + private static final String JAEGER_URL = "http://localhost:16686"; + + private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); + + @ClassRule + public static DockerComposeContainer environment = + new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) + .withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + + @BeforeAll + static void setUp() { + environment.start(); + } + + @Test + void runSampleSteps() throws Exception { + testUtils.assertTestPrerequisites(); + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.lookUpContractAgreementId(); + var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); + testUtils.assertDestinationFileContent(); + testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + assertJaegerState(); + } + + private void assertJaegerState() { + try { + var url = new URL(JAEGER_URL); + var huc = (HttpURLConnection) url.openConnection(); + assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); + } catch (IOException e) { + fail("Unable to assert Jaeger state", e); + } + } + + @AfterEach + protected void tearDown() { + testUtils.cleanTemporaryTestFiles(); + } +} diff --git a/transfer/transfer-04-open-telemetry/README.md b/transfer/transfer-04-open-telemetry/README.md index 77040306..d7bf1fb8 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/transfer/transfer-04-open-telemetry/README.md @@ -12,12 +12,7 @@ several popular [libraries and frameworks](https://github.com/open-telemetry/ope In order to visualize and analyze the traces and metrics, we use [OpenTelemetry exporters](https://opentelemetry.io/docs/instrumentation/js/exporters/) to export data into the Jaeger -tracing backend and a Prometheus endpoint. - -## Prerequisites - -Download the [opentelemetry-javaagent.jar](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar) -and place it in the root folder of this sample. +tracing backend and a Prometheus endpoint. ## Run the sample diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index db928391..5baecd25 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -1,43 +1,24 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "consumerId": "consumer", + "providerId": "provider", "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", "assetId": "test-document", "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "asset:prop:id": "test-document" - } + "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "test-document" } } } diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index ec4bae44..cd7be7fa 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -5,6 +5,7 @@ services: consumer: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: consumer OTEL_SERVICE_NAME: consumer OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -13,9 +14,12 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_DSP_PORT: 9292 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer + WEB_HTTP_DSP_PORT: 9393 WEB_HTTP_DSP_PATH: /protocol - DSP_WEBHOOK_ADDRESS: http://consumer:9292 EDC_API_AUTH_KEY: password volumes: - ../:/samples @@ -30,6 +34,7 @@ services: provider: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -37,8 +42,11 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 8182 WEB_HTTP_MANAGEMENT_PATH: /management - DSP_WEBHOOK_ADDRESS: http://provider:8282 - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt + WEB_HTTP_PROTOCOL_PORT: 8282 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol + EDC_PARTICIPANT_ID: provider + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md volumes: - ../:/samples ports: @@ -56,7 +64,6 @@ services: prometheus: image: prom/prometheus:v2.30.3 - container_name: prometheus volumes: - ./prometheus/:/etc/prometheus/ ports: diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index 3a2bfba1..d0f591e3 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -1,16 +1,17 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "dataDestination": { + "type": "File", + "path": "/samples/transfer-04-open-telemetry/README_transferred.md", + "keyName": "keyName" + }, "protocol": "dataspace-protocol-http", "assetId": "test-document", "contractId": "{agreement ID}", - "dataDestination": { - "path": "/samples/output-file.txt", - "keyName": "keyName", - "type": "File" - }, - "transferType": { - "contentType": "application/octet-stream", - "isFinite": true - }, + "connectorId": "provider", "connectorAddress": "http://provider:8282/protocol", - "connectorId": "consumer" + "privateProperties": {} } diff --git a/transfer/transfer-04-open-telemetry/input-file.txt b/transfer/transfer-04-open-telemetry/input-file.txt deleted file mode 100644 index b4f8b365..00000000 --- a/transfer/transfer-04-open-telemetry/input-file.txt +++ /dev/null @@ -1 +0,0 @@ -This is a sample file \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index ade9456d..b5c53191 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -20,16 +24,20 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) - + implementation(libs.edc.api.observability) implementation(libs.edc.configuration.filesystem) implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) implementation(libs.edc.management.api) - implementation(libs.edc.dsp) + + implementation(project(":transfer:transfer-01-file-transfer:status-checker")) + runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) @@ -43,3 +51,27 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("consumer.jar") } + +tasks.register("copyOpenTelemetryJar") { + doLast { + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } + } +} + +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 34930dd8..e9864bdd 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -19,7 +23,10 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.api.observability) @@ -43,3 +50,27 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("provider.jar") } + +tasks.register("copyOpenTelemetryJar") { + doLast { + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } + } +} + +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") +} \ No newline at end of file From 7a233cfb1922190d8acbb950e7d3ccede23ca06c Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:53:30 +0200 Subject: [PATCH 10/35] refactor: building connector jars for telemetry end to end test from docker compose file (#132) * fix: fixing wrong command for ./gradlew test in verify.yaml * fix: raising timeouts to 30s * refactor: building docker images for opentelemetry consumer & provider from dockerfile * fix: remove WEB_HTTP_DSP_PORT & WEB_HTTP_DSP_PATH from docker-compose.yaml * fix: revert temp config * docs: updating README.md for opentelemetry * docs: updating README.md for opentelemetry * refactor: using opentelemetry agent from within docker container --- .github/workflows/verify.yaml | 3 +- .../FileTransferSampleTestCommon.java | 2 +- transfer/transfer-04-open-telemetry/README.md | 36 +++++++++----- .../docker-compose.yaml | 48 ++++++++++--------- .../filetransfer.json | 2 +- .../open-telemetry-consumer/Dockerfile | 11 +++++ .../open-telemetry-consumer/build.gradle.kts | 10 ++-- .../open-telemetry-provider/Dockerfile | 11 +++++ .../open-telemetry-provider/build.gradle.kts | 11 +++-- 9 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile create mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 27141139..b6ee42e3 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,8 +46,7 @@ jobs: - name: End to End Integration Tests uses: ./.github/actions/run-tests with: - command: - - ./gradlew test -DincludeTags="EndToEndTest" + command: ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index 78e100bb..95e64ec6 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon { final String sampleAssetFilePath; final File sampleAssetFile; final File destinationFile; - Duration timeout = Duration.ofSeconds(15); + Duration timeout = Duration.ofSeconds(30); Duration pollInterval = Duration.ofMillis(500); String contractNegotiationId; diff --git a/transfer/transfer-04-open-telemetry/README.md b/transfer/transfer-04-open-telemetry/README.md index d7bf1fb8..e35677f8 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/transfer/transfer-04-open-telemetry/README.md @@ -27,7 +27,6 @@ is configured to expose a Prometheus metrics endpoint. To run the consumer, the provider, and Jaeger execute the following commands in the project root folder: ```bash -./gradlew transfer:transfer-04-open-telemetry:open-telemetry-consumer:build transfer:transfer-04-open-telemetry:open-telemetry-provider:build docker-compose -f transfer/transfer-04-open-telemetry/docker-compose.yaml up --abort-on-container-exit ``` @@ -79,26 +78,37 @@ which has to be stored in the root folder of this sample as well. The only addit ```yaml consumer: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/resources + ports: + - "9191:9191" + - "9192:9192" environment: APPLICATIONINSIGHTS_CONNECTION_STRING: APPLICATIONINSIGHTS_ROLE_NAME: consumer # optional: increase log verbosity (default level is INFO) APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL: DEBUG - WEB_HTTP_PORT: 8181 + EDC_HOSTNAME: consumer + OTEL_SERVICE_NAME: consumer + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_METRICS_EXPORTER: prometheus + WEB_HTTP_PORT: 9191 WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 8182 + WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - DSP_WEBHOOK_ADDRESS: http://consumer:8181 - volumes: - - ../:/samples - ports: - - 9191:8181 - - 9192:8182 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer + EDC_API_AUTH_KEY: password entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/applicationinsights-agent-3.2.8.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar + -javaagent:/resources/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar ``` The Application Insights Java agent will automatically collect metrics from Micrometer, without any configuration needed. diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index cd7be7fa..e5a99b6b 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -3,7 +3,14 @@ version: "3.8" services: consumer: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/resources + ports: + - "9191:9191" + - "9192:9192" environment: EDC_HOSTNAME: consumer OTEL_SERVICE_NAME: consumer @@ -18,21 +25,21 @@ services: WEB_HTTP_PROTOCOL_PATH: /protocol EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol EDC_PARTICIPANT_ID: consumer - WEB_HTTP_DSP_PORT: 9393 - WEB_HTTP_DSP_PATH: /protocol EDC_API_AUTH_KEY: password - volumes: - - ../:/samples - ports: - - "9191:9191" - - "9192:9192" entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar + -javaagent:/app/libs/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar provider: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile + volumes: + - ./:/resources + ports: + - "8181:8181" + - "8182:8182" environment: EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider @@ -46,25 +53,20 @@ services: WEB_HTTP_PROTOCOL_PATH: /protocol EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol EDC_PARTICIPANT_ID: provider - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md - volumes: - - ../:/samples - ports: - - "8181:8181" - - "8182:8182" + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /resources/README.md entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar + -javaagent:/app/libs/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar jaeger: image: jaegertracing/all-in-one ports: - - 16686:16686 + - "16686:16686" prometheus: image: prom/prometheus:v2.30.3 volumes: - ./prometheus/:/etc/prometheus/ ports: - - 9090:9090 + - "9090:9090" diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index d0f591e3..b84b7631 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -5,7 +5,7 @@ "@type": "TransferRequestDto", "dataDestination": { "type": "File", - "path": "/samples/transfer-04-open-telemetry/README_transferred.md", + "path": "/resources/README_transferred.md", "keyName": "keyName" }, "protocol": "dataspace-protocol-http", diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile new file mode 100644 index 00000000..93680a50 --- /dev/null +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-consumer:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index b5c53191..3b466313 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -24,7 +24,7 @@ plugins { } dependencies { - implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -38,6 +38,7 @@ dependencies { implementation(project(":transfer:transfer-01-file-transfer:status-checker")) + runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) @@ -54,8 +55,7 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" - val file = File(filePath) + val file = file("../opentelemetry-javaagent.jar") if (!file.exists()) { sourceSets["main"] @@ -65,13 +65,13 @@ tasks.register("copyOpenTelemetryJar") { ?.path ?.let { val sourcePath = Paths.get(it) - val targetPath = Paths.get(filePath) + val targetPath = Paths.get(file.path) Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) } } } } -tasks.compileJava { +tasks.build { finalizedBy("copyOpenTelemetryJar") } \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile new file mode 100644 index 00000000..50efc2f5 --- /dev/null +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-provider:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index e9864bdd..29fc03bf 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -23,7 +23,7 @@ plugins { } dependencies { - implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -39,6 +39,8 @@ dependencies { implementation(libs.edc.dsp) implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) + + runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -53,8 +55,7 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" - val file = File(filePath) + val file = file("../opentelemetry-javaagent.jar") if (!file.exists()) { sourceSets["main"] @@ -64,13 +65,13 @@ tasks.register("copyOpenTelemetryJar") { ?.path ?.let { val sourcePath = Paths.get(it) - val targetPath = Paths.get(filePath) + val targetPath = Paths.get(file.path) Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) } } } } -tasks.compileJava { +tasks.build { finalizedBy("copyOpenTelemetryJar") } \ No newline at end of file From 084872d5f656e94eb5980b2304eb92dff9f49953 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:09:42 +0200 Subject: [PATCH 11/35] build(deps): bump edc from 0.3.0 to 0.3.1 (#129) * build(deps): bump edc from 0.3.0 to 0.3.1 Bumps `edc` from 0.3.0 to 0.3.1. Updates `org.eclipse.edc:api-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:api-observability` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:auth-tokenbased` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:boot` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/GradlePlugins/releases) - [Commits](https://github.com/eclipse-edc/GradlePlugins/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:configuration-filesystem` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:connector-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:control-plane-api-client` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:control-plane-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:control-plane-spi` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-api` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-aws-s3` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Technology-Aws/releases) - [Commits](https://github.com/eclipse-edc/Technology-Aws/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-azure-storage` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Technology-Azure/releases) - [Commits](https://github.com/eclipse-edc/Technology-Azure/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-client` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-http` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-kafka` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-selector-api` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-selector-client` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-selector-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-spi` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:data-plane-util` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:dsp` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:http` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:iam-mock` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:jersey-micrometer` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:jetty-micrometer` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:json-ld` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:junit` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:management-api` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:micrometer-core` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:monitor-jdk-logger` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:provision-aws-s3` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Technology-Aws/releases) - [Commits](https://github.com/eclipse-edc/Technology-Aws/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:runtime-metamodel` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:transfer-data-plane` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:transfer-process-api` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:transfer-pull-http-receiver` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:transfer-spi` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:util` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:vault-azure` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Technology-Azure/releases) - [Commits](https://github.com/eclipse-edc/Technology-Azure/compare/v0.3.0...v0.3.1) Updates `org.eclipse.edc:vault-filesystem` from 0.3.0 to 0.3.1 - [Release notes](https://github.com/eclipse-edc/Connector/releases) - [Commits](https://github.com/eclipse-edc/Connector/compare/v0.3.0...v0.3.1) --- updated-dependencies: - dependency-name: org.eclipse.edc:api-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:api-observability dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:auth-tokenbased dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:boot dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:configuration-filesystem dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:connector-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:control-plane-api-client dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:control-plane-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:control-plane-spi dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-aws-s3 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-azure-storage dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-client dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-http dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-kafka dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-selector-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-selector-client dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-selector-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-spi dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:data-plane-util dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:dsp dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:http dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:iam-mock dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:jersey-micrometer dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:jetty-micrometer dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:json-ld dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:junit dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:management-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:monitor-jdk-logger dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:provision-aws-s3 dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:runtime-metamodel dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:transfer-data-plane dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:transfer-process-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:transfer-pull-http-receiver dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:transfer-spi dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:util dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:vault-azure dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.edc:vault-filesystem dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Fix compilation and tests * fix broken test setup * Fixed tests --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ndr_brt Co-authored-by: Paul Latzelsperger --- gradle/libs.versions.toml | 3 +- settings.gradle.kts | 1 - .../transfer/Transfer04openTelemetryTest.java | 8 ++-- transfer/transfer-01-file-transfer/README.md | 7 ---- .../file-transfer-consumer/build.gradle.kts | 2 - .../file-transfer-provider/build.gradle.kts | 1 + .../file-transfer-provider/config.properties | 3 ++ .../status-checker/build.gradle.kts | 21 ---------- .../checker/SampleFileStatusChecker.java | 39 ------------------ .../checker/SampleStatusCheckerExtension.java | 41 ------------------- ...rg.eclipse.edc.spi.system.ServiceExtension | 15 ------- .../listener/TransferListenerExtension.java | 19 --------- .../transfer/TransferSimulationExtension.java | 5 --- .../docker-compose.yaml | 6 +++ .../open-telemetry-consumer/build.gradle.kts | 4 +- .../open-telemetry-provider/build.gradle.kts | 3 +- 16 files changed, 19 insertions(+), 159 deletions(-) delete mode 100644 transfer/transfer-01-file-transfer/status-checker/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java delete mode 100644 transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java delete mode 100644 transfer/transfer-01-file-transfer/status-checker/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d467c6a0..1f28269e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ format.version = "1.1" [versions] assertj = "3.24.2" awaitility = "4.2.0" -edc = "0.3.0" +edc = "0.3.1" jakarta-json = "2.0.1" junit-pioneer = "2.1.0" jupiter = "5.10.0" @@ -26,6 +26,7 @@ edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-bui edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } +edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } edc-control-plane-spi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } edc-data-plane-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } diff --git a/settings.gradle.kts b/settings.gradle.kts index a8884dbb..31d39d30 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -37,7 +37,6 @@ include(":basic:basic-03-configuration") include(":transfer:transfer-01-file-transfer:file-transfer-consumer") include(":transfer:transfer-01-file-transfer:file-transfer-provider") include(":transfer:transfer-01-file-transfer:transfer-file-local") -include(":transfer:transfer-01-file-transfer:status-checker") include(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer") include(":transfer:transfer-02-file-transfer-listener:listener") diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java index eb3d7029..e562cb12 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -16,12 +16,12 @@ import org.apache.http.HttpStatus; import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.junit.ClassRule; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.io.IOException; @@ -45,9 +45,9 @@ public class Transfer04openTelemetryTest { private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - @ClassRule - public static DockerComposeContainer environment = - new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) + @Container + public static DockerComposeContainer environment = + new DockerComposeContainer<>(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) .withLocalCompose(true) .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); diff --git a/transfer/transfer-01-file-transfer/README.md b/transfer/transfer-01-file-transfer/README.md index c74b4d6d..043fe1b3 100644 --- a/transfer/transfer-01-file-transfer/README.md +++ b/transfer/transfer-01-file-transfer/README.md @@ -152,13 +152,6 @@ authentication. Therefore, we add the property `edc.api.auth.key` and set it to to configure the consumer's webhook address. We expose the DSP API endpoints on a different port and path than other endpoints, so the property `edc.dsp.callback.address` is adjusted to match the DSP API port. -The consumer connector also needs the `status-checker` extension for marking the transfer as completed on the consumer -side. - -```kotlin -implementation(project(":transfer:transfer-01-file-transfer:status-checker")) -``` - ## Run the sample Running this sample consists of multiple steps, that are executed one by one. diff --git a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts index b35e0ec2..5d9ff687 100644 --- a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts +++ b/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts @@ -34,8 +34,6 @@ dependencies { implementation(libs.edc.dsp) - implementation(project(":transfer:transfer-01-file-transfer:status-checker")) - } application { diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts index 58aa47d4..d1a7776c 100644 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts +++ b/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts @@ -22,6 +22,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.api.observability) diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties b/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties index 22d007cf..2383072c 100644 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties +++ b/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties @@ -4,7 +4,10 @@ web.http.management.port=8182 web.http.management.path=/management web.http.protocol.port=8282 web.http.protocol.path=/protocol +web.http.control.port=8283 +web.http.control.path=/control edc.samples.transfer.01.asset.path=/path/to/file edc.dsp.callback.address=http://localhost:8282/protocol edc.participant.id=provider edc.ids.id=urn:connector:provider +edc.control.endpoint=http://localhost:8283/control diff --git a/transfer/transfer-01-file-transfer/status-checker/build.gradle.kts b/transfer/transfer-01-file-transfer/status-checker/build.gradle.kts deleted file mode 100644 index 9c3cc3c6..00000000 --- a/transfer/transfer-01-file-transfer/status-checker/build.gradle.kts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2020-2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -plugins { - `java-library` -} - -dependencies { - api(libs.edc.control.plane.spi) -} diff --git a/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java b/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java deleted file mode 100644 index 9eb5268a..00000000 --- a/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2020 - 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.checker; - -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.StatusChecker; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; - -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Optional; - -public class SampleFileStatusChecker implements StatusChecker { - @Override - public boolean isComplete(TransferProcess transferProcess, List resources) { - var destination = transferProcess.getDataRequest().getDataDestination(); - var path = destination.getStringProperty("path"); - return Optional.ofNullable(path) - .map(this::checkPath) - .orElse(false); - } - - private boolean checkPath(String path) { - return Files.exists(Paths.get(path)); - } -} diff --git a/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java b/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java deleted file mode 100644 index 5e364c98..00000000 --- a/transfer/transfer-01-file-transfer/status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2020 - 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.checker; - -import org.eclipse.edc.connector.transfer.spi.status.StatusCheckerRegistry; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; - -@Extension(value = SampleStatusCheckerExtension.NAME) -public class SampleStatusCheckerExtension implements ServiceExtension { - - public static final String NAME = "Sample status checker"; - - @Inject - private StatusCheckerRegistry checkerRegistry; - - @Override - public String name() { - return NAME; - } - - - @Override - public void initialize(ServiceExtensionContext context) { - checkerRegistry.register("File", new SampleFileStatusChecker()); - } -} diff --git a/transfer/transfer-01-file-transfer/status-checker/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-01-file-transfer/status-checker/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 9e20e172..00000000 --- a/transfer/transfer-01-file-transfer/status-checker/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2020-2022 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.sample.extension.checker.SampleStatusCheckerExtension diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java b/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java index 6534e2d0..df2cf070 100644 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java +++ b/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java @@ -15,36 +15,17 @@ package org.eclipse.edc.sample.extension.listener; import org.eclipse.edc.connector.transfer.spi.observe.TransferProcessObservable; -import org.eclipse.edc.connector.transfer.spi.status.StatusCheckerRegistry; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.transfer.spi.types.StatusChecker; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; -import java.io.File; -import java.util.List; - public class TransferListenerExtension implements ServiceExtension { - @Inject - private StatusCheckerRegistry statusCheckerRegistry; - @Override public void initialize(ServiceExtensionContext context) { var transferProcessObservable = context.getService(TransferProcessObservable.class); var monitor = context.getMonitor(); transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); - statusCheckerRegistry.register("File", new FileStatusChecker()); } - private static class FileStatusChecker implements StatusChecker { - @Override - public boolean isComplete(TransferProcess transferProcess, List resources) { - var path = transferProcess.getDataDestination().getStringProperty("path"); - return path != null && new File(path).exists(); - } - } } diff --git a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java b/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java index b58416ea..eb071a9d 100644 --- a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java +++ b/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java @@ -14,7 +14,6 @@ package org.eclipse.edc.sample.extension.transfer; -import org.eclipse.edc.connector.transfer.spi.status.StatusCheckerRegistry; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.connector.transfer.spi.types.DataRequest; import org.eclipse.edc.connector.transfer.spi.types.ProvisionedDataDestinationResource; @@ -37,15 +36,11 @@ public class TransferSimulationExtension implements ServiceExtension { @Inject private TransferProcessStore store; - @Inject - private StatusCheckerRegistry statusCheckerRegistry; - @Inject private Clock clock; @Override public void initialize(ServiceExtensionContext context) { - statusCheckerRegistry.register(TEST_TYPE, (transferProcess, resources) -> false); //never completes //Insert a test TP after a delay to simulate a zombie transfer new Timer().schedule( new TimerTask() { diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index e5a99b6b..b3ad0fca 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -23,6 +23,9 @@ services: WEB_HTTP_MANAGEMENT_PATH: /management WEB_HTTP_PROTOCOL_PORT: 9292 WEB_HTTP_PROTOCOL_PATH: /protocol + WEB_HTTP_CONTROL_PORT: 9193 + WEB_HTTP_CONTROL_PATH: /control + EDC_CONTROL_ENDPOINT: http://consumer:9193/control EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol EDC_PARTICIPANT_ID: consumer EDC_API_AUTH_KEY: password @@ -51,6 +54,9 @@ services: WEB_HTTP_MANAGEMENT_PATH: /management WEB_HTTP_PROTOCOL_PORT: 8282 WEB_HTTP_PROTOCOL_PATH: /protocol + WEB_HTTP_CONTROL_PORT: 8183 + WEB_HTTP_CONTROL_PATH: /control + EDC_CONTROL_ENDPOINT: http://provider:8183/control EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol EDC_PARTICIPANT_ID: provider EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /resources/README.md diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index 3b466313..03fd684a 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -36,8 +36,6 @@ dependencies { implementation(libs.edc.management.api) implementation(libs.edc.dsp) - implementation(project(":transfer:transfer-01-file-transfer:status-checker")) - runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) @@ -74,4 +72,4 @@ tasks.register("copyOpenTelemetryJar") { tasks.build { finalizedBy("copyOpenTelemetryJar") -} \ No newline at end of file +} diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 29fc03bf..f6f67b14 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -25,6 +25,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -74,4 +75,4 @@ tasks.register("copyOpenTelemetryJar") { tasks.build { finalizedBy("copyOpenTelemetryJar") -} \ No newline at end of file +} From 861ef5f104de405ae1e45a73fd9bffd428c20722 Mon Sep 17 00:00:00 2001 From: talele08 Date: Thu, 12 Oct 2023 12:32:15 +0530 Subject: [PATCH 12/35] fix: add missing edc-control-plane-api-client dependency (#134) add missing edc-control-plane-api-client dependency --- .../http-pull-connector/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts b/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts index a16e0935..9986a7a4 100644 --- a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts +++ b/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts @@ -19,6 +19,7 @@ plugins { } dependencies { + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) From aca2e321914178112db7f9bbe1f9320417939cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:26:56 +0200 Subject: [PATCH 13/35] build(deps): bump io.opentelemetry.javaagent:opentelemetry-javaagent from 1.30.0 to 1.31.0 (#136) build(deps): bump io.opentelemetry.javaagent:opentelemetry-javaagent Bumps [io.opentelemetry.javaagent:opentelemetry-javaagent](https://github.com/open-telemetry/opentelemetry-java-instrumentation) from 1.30.0 to 1.31.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-java-instrumentation/compare/v1.30.0...v1.31.0) --- updated-dependencies: - dependency-name: io.opentelemetry.javaagent:opentelemetry-javaagent dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1f28269e..619d46be 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -74,7 +74,7 @@ testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", ve kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } -opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.30.0" +opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.31.0" [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } From 6b2493c9b13c248c19c6c3f30478243318392228 Mon Sep 17 00:00:00 2001 From: Andrea Marchini <123945788+man8pr@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:03:06 +0200 Subject: [PATCH 14/35] docs: consumer-pull transfer README update (#137) refactor: consumer-pull transfer README update - Setting up a data plane on the consumer is not required, both sink and source are on the provider side - Coherent case for paragraph titles --- .../transfer-06-consumer-pull-http/README.md | 39 +++++-------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/transfer/transfer-06-consumer-pull-http/README.md b/transfer/transfer-06-consumer-pull-http/README.md index 735f3971..23ebea8c 100644 --- a/transfer/transfer-06-consumer-pull-http/README.md +++ b/transfer/transfer-06-consumer-pull-http/README.md @@ -141,28 +141,7 @@ curl -H 'Content-Type: application/json' \ -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq ``` -### 2. Register data plane instance for consumer - -The same thing that is done for the provider must be done for the consumer - -```bash -curl -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-consumer-dataplane", - "url": "http://localhost:29192/control/transfer", - "allowedSourceTypes": [ "HttpData" ], - "allowedDestTypes": [ "HttpProxy", "HttpData" ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" - } - }' \ - -X POST "http://localhost:29193/management/v2/dataplanes" -``` - -### 3. Create an Asset on the provider side +### 2. Create an asset on the provider side The provider connector needs to transfer a file to the location specified by the consumer connector when the data are requested. In order to offer any data, the provider must maintain an internal list @@ -203,7 +182,7 @@ Additional properties on `HttpData` can be used to allow consumers to enrich the - `proxyBody`: allows attaching a body. - `proxyMethod`: allows specifying the Http Method (default `GET`) -### 4. Create a Policy on the provider +### 3. Create a policy on the provider In order to manage the accessibility rules of an asset, it is essential to create a policy. However, to keep things simple, we will choose a policy that gives direct access to all the assets that are @@ -227,7 +206,7 @@ curl -d '{ -s | jq ``` -### 5. Create a contract definition on Provider +### 4. Create a contract definition on provider To ensure an exchange between providers and consumers, the supplier must create a contract offer for the good, on the basis of which a contract agreement can be negotiated. The contract definition @@ -258,7 +237,7 @@ Sample output: } ``` -### 6. How to fetch catalog on consumer side +### 5. How to fetch catalog on consumer side In order to offer any data, the consumer can fetch the catalog from the provider, that will contain all the contract offers available for negotiation. In our case, it will contain a single contract @@ -331,7 +310,7 @@ Sample output: } ``` -### 7. Negotiate a contract +### 6. Negotiate a contract In order to request any data, a contract gets negotiated, and an agreement is resulting has to be negotiated between providers and consumers. @@ -388,7 +367,7 @@ Sample output: } ``` -### 8. Getting the contract agreement id +### 7. Getting the contract agreement id After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. This UUID is the ID of the ongoing contract negotiation between consumer and provider. The @@ -425,7 +404,7 @@ Sample output: } ``` -### 9. Start the transfer +### 8. Start the transfer As a pre-requisite, you need to have an http server that runs on port 4000 and logs all the incoming requests, it will be mandatory to get the EndpointDataReference that will be used to get the data. @@ -480,7 +459,7 @@ Sample output: } ``` -### 10. Check the transfer status +### 9. Check the transfer status Due to the nature of the transfer, it will be very fast and most likely already done by the time you read the UUID. @@ -501,7 +480,7 @@ You should see the Transfer Process in `COMPLETED` state: ``` -### 11. Pull the data +### 10. Pull the data At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed to get the data from the provider: From 946d9d8d0ad925a42d05baf903a7edf95d956acd Mon Sep 17 00:00:00 2001 From: Michael Steinert <133106264+MichaelSteinert@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:40:44 +0200 Subject: [PATCH 15/35] fix: add missing edc-control-plane-api-client dependency (#134) (#138) add missing edc-control-plane-api-client dependency --- transfer/transfer-07-provider-push-http/README.md | 10 +++++----- .../http-push-connector/build.gradle.kts | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/transfer/transfer-07-provider-push-http/README.md b/transfer/transfer-07-provider-push-http/README.md index f70fc3b9..a6769761 100644 --- a/transfer/transfer-07-provider-push-http/README.md +++ b/transfer/transfer-07-provider-push-http/README.md @@ -1,12 +1,12 @@ # Implement a simple "Provider Push" Http transfer flow -The purpose of this example is to show a data exchange between 2 connectors, one representing the -data provider and the other, the consumer. It's based on a provider push usecase that you can find +The purpose of this example is to show a data exchange between two connectors, one representing the +data provider and the other, the consumer. It's based on a provider push use case that you can find more details -on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane) +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). For the sake of simplicity, the provider and the consumer will be on the same machine, but in a real world configuration, they will likely be on different -machines. The final goal of this example is to present the steps through which the 2 connectors will +machines. The final goal of this example is to present the steps through which the two connectors will have to pass so that the consumer can have access to the data, held by the provider. Those steps are the following: @@ -297,7 +297,7 @@ looks as follows: reference Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counter offers in addition to just confirming or declining an offer. +send counteroffers in addition to just confirming or declining an offer. ```bash curl -d '{ diff --git a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts b/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts index 2961bee7..86bc1e6b 100644 --- a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts +++ b/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts @@ -24,6 +24,7 @@ repositories { dependencies { implementation(libs.edc.control.plane.core) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) implementation(libs.edc.vault.filesystem) From a334cb0f9a2d83acdba7c468a238ee6819e3efda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 08:42:41 +0200 Subject: [PATCH 16/35] build(deps): bump agilepathway/label-checker from 1.5.9 to 1.6.3 (#139) Bumps [agilepathway/label-checker](https://github.com/agilepathway/label-checker) from 1.5.9 to 1.6.3. - [Release notes](https://github.com/agilepathway/label-checker/releases) - [Commits](https://github.com/agilepathway/label-checker/compare/v1.5.9...v1.6.3) --- updated-dependencies: - dependency-name: agilepathway/label-checker dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scan-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scan-pull-request.yml b/.github/workflows/scan-pull-request.yml index 2fc5881c..8f2214fd 100644 --- a/.github/workflows/scan-pull-request.yml +++ b/.github/workflows/scan-pull-request.yml @@ -28,7 +28,7 @@ jobs: continue-on-error: false steps: - uses: actions/checkout@v4 - - uses: agilepathway/label-checker@v1.5.9 + - uses: agilepathway/label-checker@v1.6.3 with: any_of: api,bug,build,dependencies,documentation,enhancement,refactoring repo_token: ${{ secrets.GITHUB_TOKEN }} From e1d3f801c93472257251ea4cc324478d24de7ef0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:45:14 +0100 Subject: [PATCH 17/35] build(deps): bump agilepathway/label-checker from 1.6.3 to 1.6.7 (#141) Bumps [agilepathway/label-checker](https://github.com/agilepathway/label-checker) from 1.6.3 to 1.6.7. - [Release notes](https://github.com/agilepathway/label-checker/releases) - [Commits](https://github.com/agilepathway/label-checker/compare/v1.6.3...v1.6.7) --- updated-dependencies: - dependency-name: agilepathway/label-checker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scan-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scan-pull-request.yml b/.github/workflows/scan-pull-request.yml index 8f2214fd..4175ac06 100644 --- a/.github/workflows/scan-pull-request.yml +++ b/.github/workflows/scan-pull-request.yml @@ -28,7 +28,7 @@ jobs: continue-on-error: false steps: - uses: actions/checkout@v4 - - uses: agilepathway/label-checker@v1.6.3 + - uses: agilepathway/label-checker@v1.6.7 with: any_of: api,bug,build,dependencies,documentation,enhancement,refactoring repo_token: ${{ secrets.GITHUB_TOKEN }} From 3eba4910848c4dbeda3be9517c4c06afbaf6ec05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:45:25 +0100 Subject: [PATCH 18/35] build(deps): bump jupiter from 5.10.0 to 5.10.1 (#142) Bumps `jupiter` from 5.10.0 to 5.10.1. Updates `org.junit.jupiter:junit-jupiter-api` from 5.10.0 to 5.10.1 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.1) Updates `org.junit.jupiter:junit-jupiter-engine` from 5.10.0 to 5.10.1 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.1) Updates `org.junit.jupiter:junit-jupiter-params` from 5.10.0 to 5.10.1 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.1) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.junit.jupiter:junit-jupiter-params dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 619d46be..409d1069 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,7 +7,7 @@ awaitility = "4.2.0" edc = "0.3.1" jakarta-json = "2.0.1" junit-pioneer = "2.1.0" -jupiter = "5.10.0" +jupiter = "5.10.1" okhttp-mockwebserver = "5.0.0-alpha.11" openTelemetry = "1.18.0" restAssured = "5.3.2" From 94443a9acc14af6e7a26938b0f05623ed6fcb818 Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:26:45 +0100 Subject: [PATCH 19/35] refactor: revisit transfer samples (#140) * extracting negotiation tests * implementing Transfer02consumerPullTest * implementing Transfer03providerPushTest.java * implementing Transfer04eventConsumerTest.java * moving open telemetry to 'advanced' directory * reworking download of open telemetry jar inside gradle build file * checkStyle fixes * fix: clean up and merge * fix: added libs.edc.control.plane.api to build file * fix: fixed event consumer test * fix: resolving pr comments * refactor: move http-request-logger-wrapper.yaml to test directory * refactor: using docker image instead of docker compose for http request logger container * fix: fixing check style issues * fix: removing unnecessary stuff * refactor: removing HttpRequestLoggerUtil.java * fix: setting http-request-logger.jar archiveFilename explicitly --- README.md | 9 +- advanced/README.md | 12 + .../advanced-01-open-telemetry}/README.md | 88 ++- .../contract-negotiation-trace.png | Bin .../attachments/file-transfer-trace.png | Bin .../docker-compose.yaml | 93 ++++ .../open-telemetry-consumer/Dockerfile | 11 + .../open-telemetry-consumer/build.gradle.kts | 52 +- .../open-telemetry-provider/Dockerfile | 11 + .../open-telemetry-provider/build.gradle.kts | 46 +- .../prometheus/prometheus.yml | 0 .../resources}/logging.properties | 0 .../resources/negotiate-contract.json | 14 +- .../resources/start-transfer.json | 14 + build.gradle.kts | 3 +- settings.gradle.kts | 25 +- system-tests/build.gradle.kts | 15 +- .../advanced/Advanced01openTelemetryTest.java | 86 +++ .../FileTransferCommon.java} | 11 +- .../edc/samples/common/NegotiationCommon.java | 84 +++ .../samples/common/PrerequisitesCommon.java | 104 ++++ .../transfer/Transfer00prerequisitesTest.java | 39 ++ .../transfer/Transfer01fileTransferTest.java | 80 --- .../transfer/Transfer01negotiationTest.java | 53 ++ .../transfer/Transfer02consumerPullTest.java | 95 ++++ .../Transfer02fileTransferListenerTest.java | 115 ---- .../Transfer03modifyTransferProcessTest.java | 70 --- .../transfer/Transfer03providerPushTest.java | 68 +++ .../transfer/Transfer04eventConsumerTest.java | 70 +++ .../transfer/Transfer04openTelemetryTest.java | 84 --- .../streaming/Streaming01httpToHttpTest.java | 4 +- .../streaming/Streaming02KafkaToHttpTest.java | 4 +- .../util/HttpRequestLoggerConsumer.java | 36 ++ .../util/HttpRequestLoggerContainer.java | 41 ++ .../edc/samples/util/TransferUtil.java | 115 ++++ transfer/README.md | 59 +- transfer/transfer-00-prerequisites/README.md | 96 ++++ .../connector}/build.gradle.kts | 3 +- .../resources}/certs/cert.pem | 0 .../resources}/certs/cert.pfx | Bin .../resources}/certs/key.pem | 0 .../consumer-configuration.properties | 0 .../configuration}/consumer-vault.properties | 0 .../provider-configuration.properties | 0 .../configuration}/provider-vault.properties | 0 .../register-data-plane-consumer.json | 17 + .../register-data-plane-provider.json | 17 + transfer/transfer-01-file-transfer/README.md | 300 ---------- .../file-transfer-consumer/build.gradle.kts | 46 -- .../file-transfer-consumer/config.properties | 11 - .../file-transfer-provider/build.gradle.kts | 45 -- .../file-transfer-provider/config.properties | 13 - .../filetransfer.json | 17 - .../transfer-file-local/build.gradle.kts | 32 -- .../extension/api/FileTransferDataSink.java | 82 --- .../api/FileTransferDataSinkFactory.java | 65 --- .../extension/api/FileTransferDataSource.java | 54 -- .../api/FileTransferDataSourceFactory.java | 58 -- .../extension/api/FileTransferExtension.java | 108 ---- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - transfer/transfer-01-negotiation/README.md | 242 ++++++++ .../resources/create-asset.json | 18 + .../resources/create-contract-definition.json | 9 + .../resources/create-policy.json | 13 + .../resources/fetch-catalog.json | 7 + .../resources/negotiate-contract.json} | 14 +- transfer/transfer-02-consumer-pull/README.md | 131 +++++ .../resources/start-transfer.json | 14 + .../README.md | 91 --- .../build.gradle.kts | 48 -- .../config.properties | 9 - ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - .../README.md | 87 --- .../build.gradle.kts | 46 -- .../config.properties | 6 - .../simulator/build.gradle.kts | 22 - .../transfer/TransferSimulationExtension.java | 82 --- ...rg.eclipse.edc.spi.system.ServiceExtension | 16 - .../watchdog/build.gradle.kts | 23 - .../sample/extension/watchdog/Watchdog.java | 72 --- .../extension/watchdog/WatchdogExtension.java | 48 -- ...rg.eclipse.edc.spi.system.ServiceExtension | 16 - transfer/transfer-03-provider-push/README.md | 69 +++ .../resources/start-transfer.json | 15 + transfer/transfer-04-event-consumer/README.md | 112 ++++ .../consumer-with-listener}/build.gradle.kts | 11 +- .../listener/build.gradle.kts | 0 .../TransferProcessStartedListener.java} | 26 +- ...nsferProcessStartedListenerExtension.java} | 5 +- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../docker-compose.yaml | 78 --- .../filetransfer.json | 17 - .../open-telemetry-consumer/Dockerfile | 11 - .../open-telemetry-provider/Dockerfile | 11 - .../transfer-06-consumer-pull-http/README.md | 517 ------------------ .../transfer-07-provider-push-http/README.md | 445 --------------- .../certs/cert.pem | 21 - .../certs/cert.pfx | Bin 2477 -> 0 bytes .../certs/key.pem | 28 - .../consumer-configuration.properties | 17 - .../consumer-vault.properties | 1 - .../provider-configuration.properties | 18 - .../provider-vault.properties | 1 - util/http-request-logger/Dockerfile | 12 + util/http-request-logger/build.gradle.kts | 1 + 105 files changed, 1906 insertions(+), 3102 deletions(-) create mode 100644 advanced/README.md rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/README.md (69%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/attachments/contract-negotiation-trace.png (100%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/attachments/file-transfer-trace.png (100%) create mode 100644 advanced/advanced-01-open-telemetry/docker-compose.yaml create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/open-telemetry-consumer/build.gradle.kts (63%) create mode 100644 advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/open-telemetry-provider/build.gradle.kts (65%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry}/prometheus/prometheus.yml (100%) rename {transfer/transfer-04-open-telemetry => advanced/advanced-01-open-telemetry/resources}/logging.properties (100%) rename transfer/transfer-01-file-transfer/contractoffer.json => advanced/advanced-01-open-telemetry/resources/negotiate-contract.json (57%) create mode 100644 advanced/advanced-01-open-telemetry/resources/start-transfer.json create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java rename system-tests/src/test/java/org/eclipse/edc/samples/{transfer/FileTransferSampleTestCommon.java => common/FileTransferCommon.java} (96%) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java delete mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java create mode 100644 transfer/transfer-00-prerequisites/README.md rename transfer/{transfer-06-consumer-pull-http/http-pull-connector => transfer-00-prerequisites/connector}/build.gradle.kts (94%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/cert.pem (100%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/cert.pfx (100%) rename transfer/{transfer-06-consumer-pull-http => transfer-00-prerequisites/resources}/certs/key.pem (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-consumer => transfer-00-prerequisites/resources/configuration}/consumer-configuration.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-consumer => transfer-00-prerequisites/resources/configuration}/consumer-vault.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-provider => transfer-00-prerequisites/resources/configuration}/provider-configuration.properties (100%) rename transfer/{transfer-06-consumer-pull-http/http-pull-provider => transfer-00-prerequisites/resources/configuration}/provider-vault.properties (100%) create mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json create mode 100644 transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json delete mode 100644 transfer/transfer-01-file-transfer/README.md delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/file-transfer-provider/config.properties delete mode 100644 transfer/transfer-01-file-transfer/filetransfer.json delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java delete mode 100644 transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 transfer/transfer-01-negotiation/README.md create mode 100644 transfer/transfer-01-negotiation/resources/create-asset.json create mode 100644 transfer/transfer-01-negotiation/resources/create-contract-definition.json create mode 100644 transfer/transfer-01-negotiation/resources/create-policy.json create mode 100644 transfer/transfer-01-negotiation/resources/fetch-catalog.json rename transfer/{transfer-04-open-telemetry/contractoffer.json => transfer-01-negotiation/resources/negotiate-contract.json} (57%) create mode 100644 transfer/transfer-02-consumer-pull/README.md create mode 100644 transfer/transfer-02-consumer-pull/resources/start-transfer.json delete mode 100644 transfer/transfer-02-file-transfer-listener/README.md delete mode 100644 transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts delete mode 100644 transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties delete mode 100644 transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-03-modify-transferprocess/README.md delete mode 100644 transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java delete mode 100644 transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java delete mode 100644 transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 transfer/transfer-03-provider-push/README.md create mode 100644 transfer/transfer-03-provider-push/resources/start-transfer.json create mode 100644 transfer/transfer-04-event-consumer/README.md rename transfer/{transfer-07-provider-push-http/http-push-connector => transfer-04-event-consumer/consumer-with-listener}/build.gradle.kts (86%) rename transfer/{transfer-02-file-transfer-listener => transfer-04-event-consumer}/listener/build.gradle.kts (100%) rename transfer/{transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java => transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java} (52%) rename transfer/{transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java => transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java} (82%) create mode 100644 transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 transfer/transfer-04-open-telemetry/docker-compose.yaml delete mode 100644 transfer/transfer-04-open-telemetry/filetransfer.json delete mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile delete mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile delete mode 100644 transfer/transfer-06-consumer-pull-http/README.md delete mode 100644 transfer/transfer-07-provider-push-http/README.md delete mode 100644 transfer/transfer-07-provider-push-http/certs/cert.pem delete mode 100644 transfer/transfer-07-provider-push-http/certs/cert.pfx delete mode 100644 transfer/transfer-07-provider-push-http/certs/key.pem delete mode 100644 transfer/transfer-07-provider-push-http/http-push-consumer/consumer-configuration.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-consumer/consumer-vault.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-provider/provider-configuration.properties delete mode 100644 transfer/transfer-07-provider-push-http/http-push-provider/provider-vault.properties create mode 100644 util/http-request-logger/Dockerfile diff --git a/README.md b/README.md index ee72c873..865b99fb 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,12 @@ EDC and run different transfer scenarios. Click the link above to learn about th All transfer samples are located in the `transfer` directory. -### [Policy](./policy/README.md) +### [Advanced](./advanced/README.md) -These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what -configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide -custom code for their enforcement. +Collection of advanced topics regarding the EDC framework. +Click the link above to learn about the transfer samples in more detail. -All policy samples are located in the `policy` directory. +All transfer samples are located in the `advanced` directory. ## Contributing diff --git a/advanced/README.md b/advanced/README.md new file mode 100644 index 00000000..a9a645b3 --- /dev/null +++ b/advanced/README.md @@ -0,0 +1,12 @@ +# Advanced samples + +The samples in this scope teach advanced topics about the EDC framework. + +> Before starting with these samples, be sure to check out the [basic](../basic/README.md) and [transfer](../transfer/README.md) samples! + +## Samples + +### [Open Telemetry 01](./advanced-01-open-telemetry/README.md) Telemetry with OpenTelemetry and Micrometer + +In this sample you will learn how to generate traces with [OpenTelemetry](https://opentelemetry.io) +and collect and visualize them with [Jaeger](https://www.jaegertracing.io/). diff --git a/transfer/transfer-04-open-telemetry/README.md b/advanced/advanced-01-open-telemetry/README.md similarity index 69% rename from transfer/transfer-04-open-telemetry/README.md rename to advanced/advanced-01-open-telemetry/README.md index e35677f8..a88d08fa 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/advanced/advanced-01-open-telemetry/README.md @@ -1,11 +1,10 @@ # Telemetry with OpenTelemetry and Micrometer -This sample builds on top of [sample transfer-01-file-transfer](../transfer-01-file-transfer/README.md) to show how you -can: +This sample will show you how you can: -- generate traces with [OpenTelemetry](https://opentelemetry.io) and collect and visualize them with [Jaeger](https://www.jaegertracing.io/). +- generate traces with [OpenTelemetry](https://opentelemetry.io) and collect and visualize them with [Jaeger](https://www.jaegertracing.io/) - automatically collect metrics from infrastructure, server endpoints and client libraries with [Micrometer](https://micrometer.io) - and visualize them with [Prometheus](https://prometheus.io). + and visualize them with [Prometheus](https://prometheus.io) For this, this sample uses the Open Telemetry Java Agent, which dynamically injects bytecode to capture telemetry from several popular [libraries and frameworks](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation). @@ -27,27 +26,80 @@ is configured to expose a Prometheus metrics endpoint. To run the consumer, the provider, and Jaeger execute the following commands in the project root folder: ```bash -docker-compose -f transfer/transfer-04-open-telemetry/docker-compose.yaml up --abort-on-container-exit +docker-compose -f advanced/advanced-01-open-telemetry/docker-compose.yaml up --abort-on-container-exit ``` -Once the consumer and provider are up, start a contract negotiation by executing: +Open a new terminal. + +Register data planes for provider and consumer: + +```bash +curl -H 'Content-Type: application/json' \ + -H "X-Api-Key: password" \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ + -X POST "http://localhost:19193/management/v2/dataplanes" \ + -s | jq +``` ```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-04-open-telemetry/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" +curl -H 'Content-Type: application/json' \ + -H "X-Api-Key: password" \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json \ + -X POST "http://localhost:29193/management/v2/dataplanes" \ + -s | jq ``` -The contract negotiation causes an HTTP request sent from the consumer to the provider connector, followed by another -message from the provider to the consumer connector. Query the status of the contract negotiation by executing the -following command. Wait until the negotiation is in CONFIRMED state and note down the contract agreement id. +Create an asset: ```bash -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ + -s | jq ``` -Finally, update the contract agreement id in the `filetransfer.json` file and execute a file transfer with the following command: +Create a Policy on the provider connector: ```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-04-open-telemetry/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ + -s | jq +``` + +Follow up with the creation of a contract definition: + +```bash +curl -H "X-Api-Key: password" \ + -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ + -s | jq +``` + +Start a contract negotiation: + +```bash +curl -H "X-Api-Key: password" \ + -H "Content-Type: application/json" \ + -d @advanced/advanced-01-open-telemetry/resources/negotiate-contract.json \ + -X POST "http://localhost:29193/management/v2/contractnegotiations" \ + -s | jq +``` + +Wait until the negotiation is in `FINALIZED` state and call + +```bash +curl -X GET -H 'X-Api-Key: password' "http://localhost:29193/management/v2/contractnegotiations/{UUID}" +``` +to get the contract agreement id. + +Finally, update the contract agreement id in the [request body](resources/start-transfer.json) and execute a file transfer with the following command: + +```bash +curl -H "X-Api-Key: password" \ + -H "Content-Type: application/json" \ + -d @advanced/advanced-01-open-telemetry/resources/start-transfer.json \ + -X POST "http://localhost:29193/management/v2/transferprocesses" ``` You can access the Jaeger UI on your browser at `http://localhost:16686`. In the search tool, we can select the service @@ -55,10 +107,10 @@ You can access the Jaeger UI on your browser at `http://localhost:16686`. In the details on the spans contained in a trace by clicking on it in the Jaeger UI. Example contract negotiation trace: -![Contract negotiation](./attachments/contract-negotiation-trace.png) +![Contract negotiation](attachments/contract-negotiation-trace.png) Example file transfer trace: -![File transfer](./attachments/file-transfer-trace.png) +![File transfer](attachments/file-transfer-trace.png) OkHttp and Jetty are part of the [libraries and frameworks](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation) that OpenTelemetry can capture telemetry from. We can observe spans related to OkHttp and Jetty as EDC uses both @@ -80,7 +132,7 @@ which has to be stored in the root folder of this sample as well. The only addit consumer: build: context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile volumes: - ./:/resources ports: @@ -128,6 +180,4 @@ it, otherwise it will use the registered global OpenTelemetry. You can look at t [ServiceLoader documentation](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/ServiceLoader.html) to have more information about service providers. ---- - -[Previous Chapter](../transfer-03-modify-transferprocess/README.md) | [Next Chapter](../transfer-05-file-transfer-cloud/README.md) +--- \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/attachments/contract-negotiation-trace.png b/advanced/advanced-01-open-telemetry/attachments/contract-negotiation-trace.png similarity index 100% rename from transfer/transfer-04-open-telemetry/attachments/contract-negotiation-trace.png rename to advanced/advanced-01-open-telemetry/attachments/contract-negotiation-trace.png diff --git a/transfer/transfer-04-open-telemetry/attachments/file-transfer-trace.png b/advanced/advanced-01-open-telemetry/attachments/file-transfer-trace.png similarity index 100% rename from transfer/transfer-04-open-telemetry/attachments/file-transfer-trace.png rename to advanced/advanced-01-open-telemetry/attachments/file-transfer-trace.png diff --git a/advanced/advanced-01-open-telemetry/docker-compose.yaml b/advanced/advanced-01-open-telemetry/docker-compose.yaml new file mode 100644 index 00000000..27d3a792 --- /dev/null +++ b/advanced/advanced-01-open-telemetry/docker-compose.yaml @@ -0,0 +1,93 @@ +version: "3.8" + +services: + + consumer: + build: + context: ../.. + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/open-telemetry + - ../../transfer/transfer-00-prerequisites/:/prerequisites + ports: + - "29193:29193" + - "29194:29194" + environment: + EDC_HOSTNAME: consumer + OTEL_SERVICE_NAME: consumer + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_METRICS_EXPORTER: prometheus + WEB_HTTP_PORT: 29191 + WEB_HTTP_PATH: /api + WEB_HTTP_PUBLIC_PORT: 29291 + WEB_HTTP_PUBLIC_PATH: /public + WEB_HTTP_CONTROL_PORT: 29192 + WEB_HTTP_CONTROL_PATH: /control + WEB_HTTP_MANAGEMENT_PORT: 29193 + WEB_HTTP_MANAGEMENT_PATH: /management + WEB_HTTP_PROTOCOL_PORT: 29194 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_CONTROL_ENDPOINT: http://consumer:29192/control + EDC_DSP_CALLBACK_ADDRESS: http://consumer:29194/protocol + EDC_PARTICIPANT_ID: consumer + EDC_API_AUTH_KEY: password + EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx + EDC_KEYSTORE_PASSWORD: 123456 + EDC_VAULT: /prerequisites/resources/configuration/provider-vault.properties + EDC_FS_CONFIG: /prerequisites/resources/configuration/provider-configuration.properties + entrypoint: java + -javaagent:/app/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/open-telemetry/resources/logging.properties + -jar /app/connector.jar + + provider: + build: + context: ../.. + dockerfile: advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile + volumes: + - ./:/open-telemetry + - ../../transfer/transfer-00-prerequisites/:/prerequisites + ports: + - "19193:19193" + - "19192:19192" + environment: + EDC_HOSTNAME: provider + OTEL_SERVICE_NAME: provider + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + WEB_HTTP_PORT: 19191 + WEB_HTTP_PATH: /api + WEB_HTTP_PUBLIC_PORT: 19291 + WEB_HTTP_PUBLIC_PATH: /public + WEB_HTTP_CONTROL_PORT: 19192 + WEB_HTTP_CONTROL_PATH: /control + WEB_HTTP_MANAGEMENT_PORT: 19193 + WEB_HTTP_MANAGEMENT_PATH: /management + WEB_HTTP_PROTOCOL_PORT: 19194 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_CONTROL_ENDPOINT: http://provider:19192/control + EDC_DSP_CALLBACK_ADDRESS: http://provider:19194/protocol + EDC_PARTICIPANT_ID: provider + EDC_API_AUTH_KEY: password + EDC_KEYSTORE: /prerequisites/resources/certs/cert.pfx + EDC_KEYSTORE_PASSWORD: 123456 + EDC_VAULT: /prerequisites/resources/configuration/consumer-vault.properties + EDC_FS_CONFIG: /prerequisites/resources/configuration/consumer-configuration.properties + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /open-telemetry/README.md + entrypoint: java + -javaagent:/app/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/open-telemetry/resources/logging.properties + -jar /app/connector.jar + + jaeger: + image: jaegertracing/all-in-one + ports: + - "16686:16686" + + prometheus: + image: prom/prometheus:v2.30.3 + volumes: + - ./prometheus/:/etc/prometheus/ + ports: + - "9090:9090" diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile new file mode 100644 index 00000000..6641f6fa --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle advanced:advanced-01-open-telemetry:open-telemetry-consumer:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build/libs/opentelemetry-javaagent-*.jar /app/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts similarity index 63% rename from transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts rename to advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts index 03fd684a..bb8d7e95 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/advanced/advanced-01-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -26,19 +26,29 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.micrometer.core) - implementation(libs.edc.api.observability) + + implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.vault.filesystem) + implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) implementation(libs.edc.management.api) - implementation(libs.edc.dsp) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) + + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.selector.client) + + implementation(libs.edc.data.plane.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.api.observability) + implementation(libs.edc.auth.tokenbased) - runtimeOnly(libs.opentelemetry) - runtimeOnly(libs.edc.jersey.micrometer) - runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -51,25 +61,17 @@ tasks.withType { archiveFileName.set("consumer.jar") } -tasks.register("copyOpenTelemetryJar") { - doLast { - val file = file("../opentelemetry-javaagent.jar") - - if (!file.exists()) { - sourceSets["main"] - .runtimeClasspath - .files - .find { it.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get(file.path) - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } - } +tasks.register("copyOpenTelemetryJar", Copy::class) { + val openTelemetry = configurations.create("open-telemetry") + + dependencies { + openTelemetry(libs.opentelemetry) } + + from(openTelemetry) + into("build/libs") } tasks.build { finalizedBy("copyOpenTelemetryJar") -} +} \ No newline at end of file diff --git a/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile b/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile new file mode 100644 index 00000000..64bbb8bb --- /dev/null +++ b/advanced/advanced-01-open-telemetry/open-telemetry-provider/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle advanced:advanced-01-open-telemetry:open-telemetry-provider:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-provider/build/libs/opentelemetry-javaagent-*.jar /app/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/advanced/advanced-01-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts similarity index 65% rename from transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts rename to advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts index f6f67b14..be18514c 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/advanced/advanced-01-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -27,21 +27,27 @@ dependencies { implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.api.observability) + implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) + implementation(libs.edc.vault.filesystem) - implementation(libs.edc.auth.tokenbased) + implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) - implementation(libs.edc.dsp) + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.data.plane.selector.client) - implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) + implementation(libs.edc.data.plane.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + + implementation(libs.edc.api.observability) + implementation(libs.edc.auth.tokenbased) - runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -54,23 +60,15 @@ tasks.withType { archiveFileName.set("provider.jar") } -tasks.register("copyOpenTelemetryJar") { - doLast { - val file = file("../opentelemetry-javaagent.jar") - - if (!file.exists()) { - sourceSets["main"] - .runtimeClasspath - .files - .find { it.name.contains("opentelemetry-javaagent") } - ?.path - ?.let { - val sourcePath = Paths.get(it) - val targetPath = Paths.get(file.path) - Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) - } - } +tasks.register("copyOpenTelemetryJar", Copy::class) { + val openTelemetry = configurations.create("open-telemetry") + + dependencies { + openTelemetry(libs.opentelemetry) } + + from(openTelemetry) + into("build/libs") } tasks.build { diff --git a/transfer/transfer-04-open-telemetry/prometheus/prometheus.yml b/advanced/advanced-01-open-telemetry/prometheus/prometheus.yml similarity index 100% rename from transfer/transfer-04-open-telemetry/prometheus/prometheus.yml rename to advanced/advanced-01-open-telemetry/prometheus/prometheus.yml diff --git a/transfer/transfer-04-open-telemetry/logging.properties b/advanced/advanced-01-open-telemetry/resources/logging.properties similarity index 100% rename from transfer/transfer-04-open-telemetry/logging.properties rename to advanced/advanced-01-open-telemetry/resources/logging.properties diff --git a/transfer/transfer-01-file-transfer/contractoffer.json b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json similarity index 57% rename from transfer/transfer-01-file-transfer/contractoffer.json rename to advanced/advanced-01-open-telemetry/resources/negotiate-contract.json index 4313071e..5497bab7 100644 --- a/transfer/transfer-01-file-transfer/contractoffer.json +++ b/advanced/advanced-01-open-telemetry/resources/negotiate-contract.json @@ -5,20 +5,20 @@ }, "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "connectorAddress": "http://provider:19194/protocol", "consumerId": "consumer", "providerId": "provider", - "connectorAddress": "http://localhost:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", + "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "assetId": "assetId", "policy": { - "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", - "@type": "set", + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "Set", "odrl:permission": [], "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "test-document" + "odrl:target": "assetId" } } -} +} \ No newline at end of file diff --git a/advanced/advanced-01-open-telemetry/resources/start-transfer.json b/advanced/advanced-01-open-telemetry/resources/start-transfer.json new file mode 100644 index 00000000..7547b17f --- /dev/null +++ b/advanced/advanced-01-open-telemetry/resources/start-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://provider:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 32d7e2fa..83279cf4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,5 +57,4 @@ allprojects { println(sourceSets["main"].runtimeClasspath.asPath) } } - -} +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 31d39d30..3a0f500b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -34,32 +34,21 @@ include(":basic:basic-02-health-endpoint") include(":basic:basic-03-configuration") // transfer -include(":transfer:transfer-01-file-transfer:file-transfer-consumer") -include(":transfer:transfer-01-file-transfer:file-transfer-provider") -include(":transfer:transfer-01-file-transfer:transfer-file-local") +include(":transfer:transfer-00-prerequisites:connector") -include(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer") -include(":transfer:transfer-02-file-transfer-listener:listener") - -include(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer") -include(":transfer:transfer-03-modify-transferprocess:simulator") -include(":transfer:transfer-03-modify-transferprocess:watchdog") - -include(":transfer:transfer-04-open-telemetry:open-telemetry-consumer") -include(":transfer:transfer-04-open-telemetry:open-telemetry-provider") +include(":transfer:transfer-04-event-consumer:consumer-with-listener") +include(":transfer:transfer-04-event-consumer:listener") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider") include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") -include("transfer:transfer-06-consumer-pull-http:http-pull-connector") -include("transfer:transfer-07-provider-push-http:http-push-connector") - include("transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") include("transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") -include("util:http-request-logger") - +// advanced +include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") +include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") //policy @@ -71,4 +60,6 @@ include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tes // modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") +include("util:http-request-logger") + include(":system-tests") diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index 8d033245..f7f3e6a8 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -33,12 +33,13 @@ dependencies { testCompileOnly(project(":basic:basic-01-basic-connector")) testCompileOnly(project(":basic:basic-02-health-endpoint")) testCompileOnly(project(":basic:basic-03-configuration")) - testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-consumer")) - testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider")) - testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer")) - testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer")) - testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer")) - testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider")) + + testCompileOnly(project(":transfer:transfer-00-prerequisites:connector")) + testCompileOnly(project(":transfer:transfer-04-event-consumer:consumer-with-listener")) + testCompileOnly(project(":transfer:transfer-04-event-consumer:listener")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) -} + + testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-provider")) + testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-consumer")) +} \ No newline at end of file diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java new file mode 100644 index 00000000..8a4ae1ab --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/advanced/Advanced01openTelemetryTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.advanced; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.createAsset; +import static org.eclipse.edc.samples.common.NegotiationCommon.createContractDefinition; +import static org.eclipse.edc.samples.common.NegotiationCommon.createPolicy; +import static org.eclipse.edc.samples.common.NegotiationCommon.getContractAgreementId; +import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +@Testcontainers +public class Advanced01openTelemetryTest { + + private static final String DOCKER_COMPOSE_YAML = "advanced/advanced-01-open-telemetry/docker-compose.yaml"; + private static final String NEGOTIATE_CONTRACT_FILE_PATH = "advanced/advanced-01-open-telemetry/resources/negotiate-contract.json"; + private static final String START_TRANSFER_FILE_PATH = "advanced/advanced-01-open-telemetry/resources/start-transfer.json"; + private static final String JAEGER_URL = "http://localhost:16686"; + + @Container + public static DockerComposeContainer environment = + new DockerComposeContainer<>(getFileFromRelativePath(DOCKER_COMPOSE_YAML)) + .withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + + @BeforeAll + static void setUp() { + environment.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + createAsset(); + createPolicy(); + createContractDefinition(); + var contractNegotiationId = negotiateContract(NEGOTIATE_CONTRACT_FILE_PATH); + var contractAgreementId = getContractAgreementId(contractNegotiationId); + var transferProcessId = startTransfer(getFileContentFromRelativePath(START_TRANSFER_FILE_PATH), contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + assertJaegerState(); + } + + private void assertJaegerState() { + try { + var url = new URL(JAEGER_URL); + var huc = (HttpURLConnection) url.openConnection(); + assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); + } catch (IOException e) { + fail("Unable to assert Jaeger state", e); + } + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java similarity index 96% rename from system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java rename to system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java index 95e64ec6..0e79a98b 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java @@ -9,10 +9,11 @@ * * Contributors: * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases * */ -package org.eclipse.edc.samples.transfer; +package org.eclipse.edc.samples.common; import com.fasterxml.jackson.databind.ObjectMapper; import io.restassured.http.ContentType; @@ -39,7 +40,7 @@ /** * Encapsulates common settings, test steps, and helper methods for transfer samples */ -public class FileTransferSampleTestCommon { +public class FileTransferCommon { static final ObjectMapper MAPPER = new ObjectMapper(); @@ -64,7 +65,7 @@ public class FileTransferSampleTestCommon { * @param sampleAssetFilePath Relative path starting from the root of the project to a file which will be read from for transfer. * @param destinationFilePath Relative path starting from the root of the project where the transferred file will be written to. */ - public FileTransferSampleTestCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { + public FileTransferCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { this.sampleAssetFilePath = sampleAssetFilePath; sampleAssetFile = getFileFromRelativePath(sampleAssetFilePath); @@ -110,7 +111,7 @@ void cleanTemporaryTestFiles() { /** * Assert that the file to be copied exists at the expected location. - * This method waits a duration which is defined in {@link FileTransferSampleTestCommon#timeout}. + * This method waits a duration which is defined in {@link FileTransferCommon#timeout}. */ void assertDestinationFileContent() { await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() @@ -184,7 +185,7 @@ void lookUpContractAgreementId() { * Assert that a POST request to initiate transfer process is successful. * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses"} * - * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}. + * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferCommon#TRANSFER_FILE_PATH}. */ String requestTransferFile(String transferFilePath) throws IOException { var transferJsonFile = getFileFromRelativePath(transferFilePath); diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java new file mode 100644 index 00000000..d8780312 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/NegotiationCommon.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.common; + + +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.util.TransferUtil.POLL_INTERVAL; +import static org.eclipse.edc.samples.util.TransferUtil.TIMEOUT; +import static org.eclipse.edc.samples.util.TransferUtil.get; +import static org.eclipse.edc.samples.util.TransferUtil.post; + +public class NegotiationCommon { + + private static final String CREATE_ASSET_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-asset.json"; + private static final String V2_ASSETS_PATH = "/v2/assets"; + private static final String CREATE_POLICY_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-policy.json"; + private static final String V2_POLICY_DEFINITIONS_PATH = "/v2/policydefinitions"; + private static final String CREATE_CONTRACT_DEFINITION_FILE_PATH = "transfer/transfer-01-negotiation/resources/create-contract-definition.json"; + private static final String V2_CONTRACT_DEFINITIONS_PATH = "/v2/contractdefinitions"; + private static final String FETCH_CATALOG_FILE_PATH = "transfer/transfer-01-negotiation/resources/fetch-catalog.json"; + private static final String V2_CATALOG_REQUEST_PATH = "/v2/catalog/request"; + private static final String NEGOTIATE_CONTRACT_FILE_PATH = "transfer/transfer-01-negotiation/resources/negotiate-contract.json"; + private static final String V2_CONTRACT_NEGOTIATIONS_PATH = "/v2/contractnegotiations/"; + private static final String CONTRACT_NEGOTIATION_ID = "@id"; + private static final String CONTRACT_AGREEMENT_ID = "'edc:contractAgreementId'"; + + public static void createAsset() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_ASSETS_PATH, getFileContentFromRelativePath(CREATE_ASSET_FILE_PATH)); + } + + public static void createPolicy() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_POLICY_DEFINITIONS_PATH, getFileContentFromRelativePath(CREATE_POLICY_FILE_PATH)); + } + + public static void createContractDefinition() { + post(PrerequisitesCommon.PROVIDER_MANAGEMENT_URL + V2_CONTRACT_DEFINITIONS_PATH, getFileContentFromRelativePath(CREATE_CONTRACT_DEFINITION_FILE_PATH)); + } + + public static void fetchCatalog() { + post(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CATALOG_REQUEST_PATH, getFileContentFromRelativePath(FETCH_CATALOG_FILE_PATH)); + } + + public static String negotiateContract(String negotiateContractFilePath) { + var contractNegotiationId = post(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CONTRACT_NEGOTIATIONS_PATH, getFileContentFromRelativePath(negotiateContractFilePath), CONTRACT_NEGOTIATION_ID); + assertThat(contractNegotiationId).isNotEmpty(); + return contractNegotiationId; + } + + public static String negotiateContract() { + return negotiateContract(NEGOTIATE_CONTRACT_FILE_PATH); + } + + public static String getContractAgreementId(String contractNegotiationId) { + return await() + .atMost(TIMEOUT) + .pollInterval(POLL_INTERVAL) + .until(() -> get(PrerequisitesCommon.CONSUMER_MANAGEMENT_URL + V2_CONTRACT_NEGOTIATIONS_PATH + contractNegotiationId, CONTRACT_AGREEMENT_ID), Objects::nonNull); + } + + public static String runNegotiation() { + createAsset(); + createPolicy(); + createContractDefinition(); + fetchCatalog(); + var contractNegotiationId = negotiateContract(); + return getContractAgreementId(contractNegotiationId); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java new file mode 100644 index 00000000..a799da83 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.common; + +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; + +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; + +public class PrerequisitesCommon { + public static final String API_KEY_HEADER_KEY = "X-Api-Key"; + public static final String API_KEY_HEADER_VALUE = "password"; + public static final String PROVIDER_MANAGEMENT_URL = "http://localhost:19193/management"; + public static final String CONSUMER_MANAGEMENT_URL = "http://localhost:29193/management"; + public static final String CONSUMER_PUBLIC_URL = "http://localhost:29291/public"; + + private static final String CONNECTOR_MODULE_PATH = ":transfer:transfer-00-prerequisites:connector"; + private static final String PROVIDER = "provider"; + private static final String CONSUMER = "consumer"; + private static final String EDC_KEYSTORE = "edc.keystore"; + private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password"; + private static final String EDC_VAULT = "edc.vault"; + private static final String EDC_FS_CONFIG = "edc.fs.config"; + + private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx"; + private static final String KEYSTORE_PASSWORD = "123456"; + private static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties"; + private static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties"; + private static final String PROVIDER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties"; + private static final String CONSUMER_VAULT_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties"; + private static final String REGISTER_DATA_PLANE_PROVIDER_JSON = "transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json"; + private static final String REGISTER_DATA_PLANE_CONSUMER_JSON = "transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json"; + private static final String V2_DATAPLANES_PATH = "/v2/dataplanes"; + + public static EdcRuntimeExtension getProvider() { + return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_VAULT_PROPERTIES_FILE_PATH, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); + + } + + public static EdcRuntimeExtension getConsumer() { + return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + } + + public static EdcRuntimeExtension getConsumer(String modulePath) { + return getConnector(modulePath, CONSUMER, CONSUMER_VAULT_PROPERTIES_FILE_PATH, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + } + + public static void registerDataPlaneProvider() { + registerDataPlane(PROVIDER_MANAGEMENT_URL, REGISTER_DATA_PLANE_PROVIDER_JSON); + } + + public static void runPrerequisites() { + registerDataPlaneProvider(); + } + + private static EdcRuntimeExtension getConnector( + String modulePath, + String moduleName, + String vaultPropertiesFilePath, + String configPropertiesFilePath + ) { + return new EdcRuntimeExtension( + modulePath, + moduleName, + Map.of( + EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(), + EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD, + EDC_VAULT, getFileFromRelativePath(vaultPropertiesFilePath).getAbsolutePath(), + EDC_FS_CONFIG, getFileFromRelativePath(configPropertiesFilePath).getAbsolutePath() + ) + ); + } + + private static void registerDataPlane(String host, String payloadPath) { + var requestBody = getFileFromRelativePath(payloadPath); + + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(host + V2_DATAPLANES_PATH) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java new file mode 100644 index 00000000..b05bb404 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer00prerequisitesTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.registerDataPlaneProvider; + +@EndToEndTest +public class Transfer00prerequisitesTest { + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Test + void runSampleSteps() { + registerDataPlaneProvider(); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java deleted file mode 100644 index f3cf5a53..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01fileTransferTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Map; - -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; - -@EndToEndTest -public class Transfer01fileTransferTest { - - private static final String SAMPLE_FOLDER = "transfer/transfer-01-file-transfer"; - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = SAMPLE_FOLDER + "/file-transfer-consumer/config.properties"; - static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = SAMPLE_FOLDER + "/file-transfer-provider/config.properties"; - static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/requested.test.txt"; - - @RegisterExtension - static EdcRuntimeExtension provider = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-provider", - "provider", - Map.of( - // Override 'edc.samples.transfer.01.asset.path' implicitly set via property 'edc.fs.config'. - "edc.samples.transfer.01.asset.path", getFileFromRelativePath(SAMPLE_ASSET_FILE_PATH).getAbsolutePath(), - "edc.fs.config", getFileFromRelativePath(PROVIDER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() throws Exception { - testUtils.assertTestPrerequisites(); - - testUtils.initiateContractNegotiation(); - testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(); - testUtils.assertDestinationFileContent(); - - testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); - } - - @AfterEach - protected void tearDown() { - testUtils.cleanTemporaryTestFiles(); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java new file mode 100644 index 00000000..b9b10f15 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer01negotiationTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.NegotiationCommon.createAsset; +import static org.eclipse.edc.samples.common.NegotiationCommon.createContractDefinition; +import static org.eclipse.edc.samples.common.NegotiationCommon.createPolicy; +import static org.eclipse.edc.samples.common.NegotiationCommon.fetchCatalog; +import static org.eclipse.edc.samples.common.NegotiationCommon.getContractAgreementId; +import static org.eclipse.edc.samples.common.NegotiationCommon.negotiateContract; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; + +@EndToEndTest +public class Transfer01negotiationTest { + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Test + void runSampleSteps() { + runPrerequisites(); + createAsset(); + createPolicy(); + createContractDefinition(); + fetchCatalog(); + var contractNegotiationId = negotiateContract(); + var contractAgreementId = getContractAgreementId(contractNegotiationId); + assertThat(contractAgreementId).isNotEmpty(); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java new file mode 100644 index 00000000..49523807 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02consumerPullTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerConsumer; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static io.restassured.RestAssured.given; +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_PUBLIC_URL; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.not; + +@EndToEndTest +@Testcontainers +public class Transfer02consumerPullTest { + + private static final HttpRequestLoggerConsumer LOG_CONSUMER = new HttpRequestLoggerConsumer(); + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-02-consumer-pull/resources/start-transfer.json"; + private static final String AUTH_CODE_KEY = "authCode"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Container + public static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(LOG_CONSUMER); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var contractAgreementId = runNegotiation(); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + var authCode = LOG_CONSUMER.getJsonValue(AUTH_CODE_KEY); + checkData(authCode); + } + + private static void checkData(String authCode) { + var result = given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE, AUTHORIZATION, authCode) + .when() + .get(CONSUMER_PUBLIC_URL) + .then() + .statusCode(HttpStatus.SC_OK) + .log() + .ifError() + .body("[0].name", not(emptyString())) + .extract() + .jsonPath() + .get("[0].name"); + + assertThat(result).isEqualTo("Leanne Graham"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java deleted file mode 100644 index 8de5103b..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer02fileTransferListenerTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.io.File; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; - -@EndToEndTest -public class Transfer02fileTransferListenerTest { - - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties"; - static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-01-file-transfer/file-transfer-provider/config.properties"; - // Reuse an already existing file for the test. Could be set to any other existing file in the repository. - static final String SAMPLE_ASSET_FILE_PATH = "transfer/transfer-02-file-transfer-listener/README.md"; - static final String DESTINATION_FILE_PATH = "transfer/transfer-02-file-transfer-listener/requested.test.txt"; - // marker.txt is a fixed name and always in the same directory as the file defined with DESTINATION_FILE_PATH. - static final String MARKER_FILE_PATH = "transfer/transfer-02-file-transfer-listener/marker.txt"; - static final File MARKER_FILE = getFileFromRelativePath(MARKER_FILE_PATH); - static final String MARKER_FILE_CONTENT = "Transfer complete"; - @RegisterExtension - static EdcRuntimeExtension provider = new EdcRuntimeExtension( - ":transfer:transfer-01-file-transfer:file-transfer-provider", - "provider", - Map.of( - // Override 'edc.samples.transfer.01.asset.path' implicitly set via property 'edc.fs.config'. - "edc.samples.transfer.01.asset.path", getFileFromRelativePath(SAMPLE_ASSET_FILE_PATH).getAbsolutePath(), - "edc.fs.config", getFileFromRelativePath(PROVIDER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - private final FileTransferSampleTestCommon common = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - @AfterEach - void tearDown() { - cleanTemporaryTestFiles(); - } - - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() throws Exception { - assertTestPrerequisites(); - - common.initiateContractNegotiation(); - common.lookUpContractAgreementId(); - common.requestTransferFile(); - common.assertDestinationFileContent(); - assertFileContent(MARKER_FILE, MARKER_FILE_CONTENT); - } - - /** - * Assert that prerequisites are fulfilled before running the test. - * This assertion checks only whether the file to be copied is not existing already. - */ - void assertTestPrerequisites() { - common.assertTestPrerequisites(); - - assertThat(MARKER_FILE).doesNotExist(); - } - - /** - * Remove files created while running the tests. - * The copied file and the marker file will be deleted. - */ - void cleanTemporaryTestFiles() { - common.cleanTemporaryTestFiles(); - - MARKER_FILE.delete(); - } - - /** - * Assert that the marker file has been created at the expected location with the expected content. - * This method waits a duration which is defined in {@link FileTransferSampleTestCommon#timeout}. - */ - void assertFileContent(File markerFile, @NotNull String markerFileContent) { - await().atMost(common.timeout).pollInterval(common.pollInterval).untilAsserted(() - -> assertThat(markerFile).hasContent(markerFileContent)); - } - - -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java deleted file mode 100644 index ebd2eab2..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03modifyTransferProcessTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial test implementation for sample - * - */ - -package org.eclipse.edc.samples.transfer; - -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.time.Duration; -import java.util.Map; - -import static io.restassured.RestAssured.given; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.API_KEY_HEADER_KEY; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.API_KEY_HEADER_VALUE; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.MANAGEMENT_API_URL; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; -import static org.hamcrest.CoreMatchers.is; - -@EndToEndTest -public class Transfer03modifyTransferProcessTest { - - static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties"; - static final Duration DURATION = Duration.ofSeconds(15); - static final Duration POLL_INTERVAL = Duration.ofMillis(500); - - @RegisterExtension - static EdcRuntimeExtension consumer = new EdcRuntimeExtension( - ":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer", - "consumer", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONSUMER_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - /** - * Requests transfer processes from management API and check for expected changes on the transfer process. - */ - @Test - void runSample() { - await().atMost(DURATION).pollInterval(POLL_INTERVAL).untilAsserted(() -> { - given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .contentType(ContentType.JSON) - .post(MANAGEMENT_API_URL + "/v2/transferprocesses/request") - .then() - .statusCode(HttpStatus.SC_OK) - .contentType(ContentType.JSON) - .body("[0].@id", is("tp-sample-transfer-03")) - .body("[0].'edc:state'", is("TERMINATING")) - .body("[0].'edc:errorDetail'", is("timeout by watchdog")); - }); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java new file mode 100644 index 00000000..a607c159 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer03providerPushTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerConsumer; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +@Testcontainers +public class Transfer03providerPushTest { + + private static final HttpRequestLoggerConsumer LOG_CONSUMER = new HttpRequestLoggerConsumer(); + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-03-provider-push/resources/start-transfer.json"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(); + + @Container + public static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(LOG_CONSUMER); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + runPrerequisites(); + var contractAgreementId = runNegotiation(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.COMPLETED); + assertThat(LOG_CONSUMER.toUtf8String()).contains("Leanne Graham"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java new file mode 100644 index 00000000..60f42781 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04eventConsumerTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.samples.util.HttpRequestLoggerContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.runPrerequisites; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; + +@EndToEndTest +public class Transfer04eventConsumerTest { + private static final String CONSUMER_WITH_LISTENER_MODULE_PATH = ":transfer:transfer-04-event-consumer:consumer-with-listener"; + private static final String START_TRANSFER_FILE_PATH = "transfer/transfer-02-consumer-pull/resources/start-transfer.json"; + + @RegisterExtension + static EdcRuntimeExtension provider = getProvider(); + + @RegisterExtension + static EdcRuntimeExtension consumer = getConsumer(CONSUMER_WITH_LISTENER_MODULE_PATH); + + @Container + static HttpRequestLoggerContainer httpRequestLoggerContainer = new HttpRequestLoggerContainer(); + + @BeforeAll + static void setUp() { + httpRequestLoggerContainer.start(); + } + + @Test + void runSampleSteps() { + var standardOutputStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(standardOutputStream)); + runPrerequisites(); + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var contractAgreementId = runNegotiation(); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + var standardOutput = standardOutputStream.toString(); + assertThat(standardOutput).contains("TransferProcessStartedListener received STARTED event"); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java deleted file mode 100644 index e562cb12..00000000 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Mercedes-Benz Tech Innovation GmbH - Sample workflow test - * - */ - -package org.eclipse.edc.samples.transfer; - -import org.apache.http.HttpStatus; -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; - -@EndToEndTest -@Testcontainers -public class Transfer04openTelemetryTest { - - private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; - private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; - private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; - private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; - private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; - private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; - private static final String JAEGER_URL = "http://localhost:16686"; - - private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); - - @Container - public static DockerComposeContainer environment = - new DockerComposeContainer<>(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) - .withLocalCompose(true) - .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); - - @BeforeAll - static void setUp() { - environment.start(); - } - - @Test - void runSampleSteps() throws Exception { - testUtils.assertTestPrerequisites(); - testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - testUtils.lookUpContractAgreementId(); - var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); - testUtils.assertDestinationFileContent(); - testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); - assertJaegerState(); - } - - private void assertJaegerState() { - try { - var url = new URL(JAEGER_URL); - var huc = (HttpURLConnection) url.openConnection(); - assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); - } catch (IOException e) { - fail("Unable to assert Jaeger state", e); - } - } - - @AfterEach - protected void tearDown() { - testUtils.cleanTemporaryTestFiles(); - } -} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java index 0259d0f1..12b70552 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming01httpToHttpTest.java @@ -37,8 +37,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileContentFromRelativePath; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; @EndToEndTest public class Streaming01httpToHttpTest { diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java index 21708542..de876440 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming02KafkaToHttpTest.java @@ -44,8 +44,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileContentFromRelativePath; -import static org.eclipse.edc.samples.transfer.FileTransferSampleTestCommon.getFileFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; @Testcontainers @EndToEndTest diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java new file mode 100644 index 00000000..687612d7 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerConsumer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import org.testcontainers.containers.output.ToStringConsumer; + +import java.util.regex.Pattern; + +import static java.lang.String.format; +import static org.apache.commons.lang3.StringUtils.EMPTY; + +public class HttpRequestLoggerConsumer extends ToStringConsumer { + + private static final String REGEX_FORMAT = "(?<=\"%s\":\")[^\"]*"; + + public String getJsonValue(String key) { + var pattern = Pattern.compile(format(REGEX_FORMAT, key)); + var matcher = pattern.matcher(toUtf8String()); + if (matcher.find()) { + return matcher.group(); + } + return EMPTY; + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java new file mode 100644 index 00000000..9f7a7792 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/HttpRequestLoggerContainer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.ToStringConsumer; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.util.List; + +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; + +public class HttpRequestLoggerContainer extends GenericContainer { + + private static final String HTTP_REQUEST_LOGGER_DOCKERFILE_PATH = "util/http-request-logger/Dockerfile"; + private static final ImageFromDockerfile IMAGE_FROM_DOCKERFILE = new ImageFromDockerfile() + .withDockerfile(getFileFromRelativePath(HTTP_REQUEST_LOGGER_DOCKERFILE_PATH).toPath()); + private static final String PORT_BINDING = "4000:4000"; + + public HttpRequestLoggerContainer() { + super(IMAGE_FROM_DOCKERFILE); + this.setPortBindings(List.of(PORT_BINDING)); + } + + public HttpRequestLoggerContainer(ToStringConsumer toStringConsumer) { + this(); + this.setLogConsumers(List.of(toStringConsumer)); + } +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java new file mode 100644 index 00000000..34189abe --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/util/TransferUtil.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Initial implementation + * + */ + +package org.eclipse.edc.samples.util; + +import io.restassured.http.ContentType; +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; + +import java.time.Duration; + +import static io.restassured.RestAssured.given; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_MANAGEMENT_URL; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.not; + +public class TransferUtil { + + public static final Duration TIMEOUT = Duration.ofSeconds(30); + public static final Duration POLL_DELAY = Duration.ofMillis(1000); + public static final Duration POLL_INTERVAL = Duration.ofMillis(500); + + private static final String TRANSFER_PROCESS_ID = "@id"; + private static final String CONTRACT_AGREEMENT_ID_KEY = ""; + private static final String V2_TRANSFER_PROCESSES_PATH = "/v2/transferprocesses/"; + private static final String EDC_STATE = "'edc:state'"; + + public static void get(String url) { + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .when() + .get(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } + + public static String get(String url, String jsonPath) { + return given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .when() + .get(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK) + .body(jsonPath, not(emptyString())) + .extract() + .jsonPath() + .get(jsonPath); + } + + public static void post(String url, String requestBody) { + given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK); + } + + public static String post(String url, String requestBody, String jsonPath) { + return given() + .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .contentType(ContentType.JSON) + .body(requestBody) + .when() + .post(url) + .then() + .log() + .ifError() + .statusCode(HttpStatus.SC_OK) + .body(jsonPath, not(emptyString())) + .extract() + .jsonPath() + .get(jsonPath); + } + + public static String startTransfer(String requestBody, String contractAgreementId) { + requestBody = requestBody.replaceAll(CONTRACT_AGREEMENT_ID_KEY, contractAgreementId); + return post(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH, requestBody, TRANSFER_PROCESS_ID); + } + + public static void checkTransferStatus(String transferProcessId, TransferProcessStates status) { + await() + .atMost(TIMEOUT) + .pollDelay(POLL_DELAY) + .pollInterval(POLL_INTERVAL) + .until( + () -> get(CONSUMER_MANAGEMENT_URL + V2_TRANSFER_PROCESSES_PATH + transferProcessId, EDC_STATE), + (result) -> status.name().equals(result) + ); + } +} diff --git a/transfer/README.md b/transfer/README.md index a6e2b524..de66133b 100644 --- a/transfer/README.md +++ b/transfer/README.md @@ -2,38 +2,39 @@ The samples in this scope revolve around the topic of transferring data between two connectors. Here you will learn about the steps required for a transfer on provider as well as consumer side. The -samples start with the simple example of a local file transfer and then show different ways to tweak -that transfer, before a transfer is performed between different cloud providers. +samples start with the simple example of a building and configuring the connector library. +Furthermore, you will learn about the contract negotiation workflow and how to perform a data transfer. > Before starting with these samples, be sure to check out the [basic samples](../basic/README.md)! ## Samples -### [Transfer sample 01](./transfer-01-file-transfer/README.md): Perform a local file transfer +### [Transfer sample 00](./transfer-00-prerequisites/README.md): Prerequisites -In this sample you will perform your first data transfer. To keep it simple, a file is transferred -on your local machine from one directory to another. You will see which extensions and -configurations are required for a transfer and learn -how to create a data offer as a provider as well as which steps to perform as a consumer. +This is a prerequisite for the following chapters. +You will build the connector library here, configure and execute it. -### [Transfer sample 02](./transfer-02-file-transfer-listener/README.md): Implement a transfer listener +### [Transfer sample 01](./transfer-01-negotiation/README.md): Negotiation -As you'll learn in the first transfer sample, the process of a data transfer is executed in a state -machine and runs asynchronously in the background after being initiated. This sample is an -enhancement of the previous sample and shows how a listener can be used to immediately react to -state changes in this asynchronous process. +Before two connectors can exchange actual data, negotiation has to take place. +The final goal of this example is to showcase the negotiation workflow between two connectors so that +the actual data transfer can take place. This chapter is a prerequisite to the following chapters. -### [Transfer sample 03](./transfer-03-modify-transferprocess/README.md): Modify a TransferProcess +### [Transfer sample 02](./transfer-02-consumer-pull/README.md): Perform a consumer pull exchange between a consumer and a provider -This sample is another enhancement of the first transfer sample. After you've learned how to react -to state changes during a data transfer, here you will see how to actively modify a process in the -state machine in a thread-safe manner. +In this sample you will perform your first actual data transfer. +The purpose of this sample is to show a data exchange between 2 connectors, one representing the +data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find +more details on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). -### [Transfer sample 04](./transfer-04-open-telemetry/README.md): Open Telemetry +### [Transfer sample 03](transfer-03-provider-push/README.md): Perform a provider push exchange between a consumer and a provider -Now that you've gotten familiar with the process of transferring data, this sample will show -how `OpenTelemetry`,`Jaeger`, `Prometheus` and `Micrometer` can be used to collect and visualize -traces and metrics during this process. +This sample demonstrates the "provider push" use case that you can find more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +### [Transfer sample 04](transfer-04-event-consumer/README.md): Consuming connector events + +In this sample you will learn how to react to certain connector events. ### [Transfer sample 05](./transfer-05-file-transfer-cloud/README.md): Perform a file transfer between cloud providers @@ -43,20 +44,4 @@ transfer scenario, where a file is transferred not in the local file system, but different cloud providers. In this sample you will set up a provider that offers a file located in an `Azure Blob Storage`, and a consumer that requests to transfer this file to an `AWS S3 bucket`. Terraform is used for creating all required cloud -resources. - -### [Transfer sample 06](./transfer-06-consumer-pull-http/README.md): Perform a consumer pull exchange between a consumer and a provider - -In this sample, we will describe a step-by-step guide to demonstrate a consumer pull exchange -between two connections. One connecter is a consumer and the other is a provider. The consumer will -initiate a transfer, and the provider will send an EndpointDataReference to the consumer. Finally, -the consumer will be able to access the data by requesting the endpoint that received through the -EndpointDataReference. - -### [Transfer sample 07](./transfer-07-provider-push-http/README.md): Perform a provider push exchange between a consumer and a provider - -In this sample, we will describe a step-by-step guide to demonstrate a provider push exchange -between two connections. One connecter is a consumer and the other is a provider. The consumer will -initiate the transfer by sending a DataRequest with any destination type other -than HttpProxy, and the provider will fetch the date from the actual DataSource and push it to the -consumer. +resources. \ No newline at end of file diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md new file mode 100644 index 00000000..4e8a968d --- /dev/null +++ b/transfer/transfer-00-prerequisites/README.md @@ -0,0 +1,96 @@ +# Prerequisites + +The purpose of this example is to make preparations for a connector-to-connector communication. +For that we'll set up two connectors. + +One connector will be the "consumer" connector while the other will act as the +"provider" connector. More on that in the following chapters. + +For the sake of simplicity, the provider and the consumer +will run on the same machine, but in a real world configuration, they will likely be on different +machines. + +This sample will go through: + +* Building the connector module +* Running the provider connector +* Running the consumer connector +* Registering data plane instance for the provider connector + +## Run the sample + +### 1. Build the connector + +When we speak of a connector we actually mean a .jar file that is launched on a machine. +Before we can launch a connector we'll have to build the .jar file. + +Execute this command in project root: + +```bash +./gradlew transfer:transfer-00-prerequisites:connector:build +``` + +After the build end you should verify that the connector.jar is created in the directory +[/connector/build/libs/connector.jar](connector/build/libs/connector.jar) + +We can use the same .jar file for both connectors. Note that the consumer and provider connectors differ in their configuration. + +Inspect the different configuration files below: + +* [provider-configuration.properties](resources/configuration/provider-configuration.properties) +* [consumer-configuration.properties](resources/configuration/consumer-configuration.properties) + +The section bellow will show you some explanation about some of the properties that you can find in +the configuration files. + +#### 1. edc.receiver.http.endpoint + +This property is used to define the endpoint where the connector consumer will send the +EndpointDataReference. + +#### 2. edc.dataplane.token.validation.endpoint + +This property is used to define the endpoint exposed by the control plane to validate the token. + +### 2. Run the connectors + +To run the provider, just run the following command + +```bash +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +``` + +To run the consumer, just run the following command (different terminal) + +```bash +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +``` + +Assuming you didn't change the ports in config files, the consumer will listen on the +ports `29191`, `29192` (management API) and `29292` (DSP API) and the provider will listen on the +ports `12181`, `19182` (management API) and `19282` (DSP API). + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 3. Register data plane instance for provider + +Before a consumer can start talking to a provider, it is necessary to register the data plane +instance of a connector. This is done by sending a POST request to the management API of the +provider connector. The [request body](resources/dataplane/register-data-plane-provider.json) should contain the data plane instance of the consumer +connector. + +The registration of the provider data plane instance is done by sending a POST +request to the management API of the connector. + +Open a new terminal and execute: + +```bash +curl -H 'Content-Type: application/json' \ + -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ + -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq +``` + +The connectors have been configured successfully and are ready to be used. + +[Next Chapter](../transfer-01-negotiation/README.md) \ No newline at end of file diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts b/transfer/transfer-00-prerequisites/connector/build.gradle.kts similarity index 94% rename from transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts rename to transfer/transfer-00-prerequisites/connector/build.gradle.kts index 9986a7a4..216f0f6e 100644 --- a/transfer/transfer-06-consumer-pull-http/http-pull-connector/build.gradle.kts +++ b/transfer/transfer-00-prerequisites/connector/build.gradle.kts @@ -20,6 +20,7 @@ plugins { dependencies { implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) implementation(libs.edc.control.plane.core) implementation(libs.edc.dsp) implementation(libs.edc.configuration.filesystem) @@ -47,6 +48,6 @@ var distZip = tasks.getByName("distZip") tasks.withType { mergeServiceFiles() - archiveFileName.set("pull-connector.jar") + archiveFileName.set("connector.jar") dependsOn(distTar, distZip) } diff --git a/transfer/transfer-06-consumer-pull-http/certs/cert.pem b/transfer/transfer-00-prerequisites/resources/certs/cert.pem similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/cert.pem rename to transfer/transfer-00-prerequisites/resources/certs/cert.pem diff --git a/transfer/transfer-06-consumer-pull-http/certs/cert.pfx b/transfer/transfer-00-prerequisites/resources/certs/cert.pfx similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/cert.pfx rename to transfer/transfer-00-prerequisites/resources/certs/cert.pfx diff --git a/transfer/transfer-06-consumer-pull-http/certs/key.pem b/transfer/transfer-00-prerequisites/resources/certs/key.pem similarity index 100% rename from transfer/transfer-06-consumer-pull-http/certs/key.pem rename to transfer/transfer-00-prerequisites/resources/certs/key.pem diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties rename to transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties rename to transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties rename to transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties diff --git a/transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties b/transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties similarity index 100% rename from transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties rename to transfer/transfer-00-prerequisites/resources/configuration/provider-vault.properties diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json new file mode 100644 index 00000000..73045bc6 --- /dev/null +++ b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-consumer.json @@ -0,0 +1,17 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "http-pull-consumer-dataplane", + "url": "http://localhost:29192/control/transfer", + "allowedSourceTypes": [ + "HttpData" + ], + "allowedDestTypes": [ + "HttpProxy", + "HttpData" + ], + "properties": { + "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" + } +} \ No newline at end of file diff --git a/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json new file mode 100644 index 00000000..6b90e706 --- /dev/null +++ b/transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json @@ -0,0 +1,17 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "http-pull-provider-dataplane", + "url": "http://localhost:19192/control/transfer", + "allowedSourceTypes": [ + "HttpData" + ], + "allowedDestTypes": [ + "HttpProxy", + "HttpData" + ], + "properties": { + "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" + } +} \ No newline at end of file diff --git a/transfer/transfer-01-file-transfer/README.md b/transfer/transfer-01-file-transfer/README.md deleted file mode 100644 index 043fe1b3..00000000 --- a/transfer/transfer-01-file-transfer/README.md +++ /dev/null @@ -1,300 +0,0 @@ -# Implement a simple file transfer - -After successfully providing custom configuration properties to the EDC, we will perform a data transfer next: transmit -a test file from one connector to another connector. We want to keep things simple, so we will run both connectors on -the same physical machine (i.e. your development machine) and the file is transferred from one folder in the file system -to another folder. It is not difficult to imagine that instead of the local file system, the transfer happens between -more sophisticated storage locations, like a database or a cloud storage. - -This is quite a big step up from the previous sample, where we ran only one connector. Those are the concrete tasks: - -* Creating an additional connector, so that in the end we have two connectors, a consumer and a provider -* Providing communication between provider and consumer using DSP messages -* Utilizing the management API to interact with the connector system -* Performing a contract negotiation between provider and consumer -* Performing a file transfer - * The consumer will initiate a file transfer - * The provider will fulfill that request and copy a file to the desired location - -Also, in order to keep things organized, the code in this example has been separated into several Java modules: - -* `file-transfer-[consumer|provider]`: contains the configuration and build files for both the consumer and the provider connector -* `transfer-file-local`: contains all the code necessary for the file transfer, integrated on provider side -* `status-checker`: contains the code for checking if the file has been transfer, integrated on the consumer side - -## Create the file transfer extension - -The provider connector needs to transfer a file to the location specified by the consumer connector when the data is -requested. In order to offer any data, the provider must maintain an internal list of assets that are available for -transfer, the so-called "catalog". For the sake of simplicity we use an in-memory catalog and pre-fill it with just one -single class. The provider also needs to create a contract offer for the asset, based on which a contract agreement can -be negotiated. For this, we also use an in-memory store and add a single contract definition that is valid for the -asset. - -```java -// in FileTransferExtension.java -@Override -public void initialize(ServiceExtensionContext context){ - // ... - var policy = createPolicy(); - policyStore.save(policy); - - registerDataEntries(context); - registerContractDefinition(policy.getUid()); - // ... -} - -//... - -private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); - var assetPath = Path.of(assetPathSetting); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("path", assetPath.getParent().toString()) - .property("filename", assetPath.getFileName().toString()) - .build(); - - var assetId = "test-document"; - var asset = Asset.Builder.newInstance().id(assetId).build(); - - assetIndex.create(asset, dataAddress); - } - -private void registerContractDefinition(String uid) { - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(uid) - .contractPolicyId(uid) - .assetsSelectorCriterion(criterion(Asset.PROPERTY_ID, "=", "1")) - .whenEquals(Asset.PROPERTY_ID, "test-document") - .build()) - .build(); - - contractStore.save(contractDefinition); - } -``` - -This adds an `Asset` to the `AssetIndex` and the relative `DataAddress` to the `DataAddressResolver`. -Or, in other words, your provider now "hosts" one file named `test-document.txt` located in the path -configured by the setting `edc.samples.transfer.01.asset.path` on your development machine. It makes it available for -transfer under its `id` `"test-document"`. While it makes sense to have some sort of similarity between file name and -id, it is by no means mandatory. - -It also adds a `ContractDefinition` with `id` `1` and a previously created `Policy` (code omitted above), that poses no -restrictions on the data usage. The `ContractDefinition` also has an `assetsSelector` `Criterion` defining that it is -valid for all assets with the `id` `test-document`. Thus, it is valid for the created asset. - -Next to offering the file, the provider also needs to be able to transfer the file. Therefore, the `transfer-file` -module also provides the code for copying the file to a specified location (code omitted here for brevity). It contains -the [FileTransferDataSource](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java) -and the [FileTransferDataSink](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java) -as well as respective factories for both. The factories are registered with the `PipelineService` in the -[FileTransferExtension](transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java), -thus making them available when a data request is processed. - -## Create the status checker extension - -The consumer needs to know when the file transfer has been completed. For doing that, in the extension -we are going to implement a custom `StatusChecker` that will be registered with the `StatusCheckerRegistry` in the -[SampleStatusCheckerExtension](status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleStatusCheckerExtension.java) -The custom status checker will handle the check for the destination type `File` and it will check that the path -specified in the data requests exists. The code is available in the -class [SampleFileStatusChecker](status-checker/src/main/java/org/eclipse/edc/sample/extension/checker/SampleFileStatusChecker.java) - -## Create the connectors - -After creating the required extensions, we next need to create the two connectors. For both of them we need a gradle -build file and a config file. Common dependencies we need to add to the build files on both sides are the following: - -```kotlin -// in file-transfer-consumer/build.gradle.kts and file-transfer-provider/build.gradle.kts: -implementation(libs.edc.configuration.filesystem) - -implementation(libs.edc.dsp) -implementation(libs.edc.iam.mock) - -implementation(libs.edc.management.api) -implementation(libs.edc.auth.tokenbased) -``` - -Three of these dependencies are new and have not been used in the previous samples: -1. `data-protocols:dsp`: contains all DSP modules and therefore enables DSP communication with other connectors -2. `extensions:iam:iam-mock`: provides a no-op identity provider, which does not require certificates and performs no checks -3. `extensions:api:auth-tokenbased`: adds authentication for management API endpoints - -### Provider connector - -As the provider connector is the one performing the file transfer after the file has been requested by the consumer, it -needs the `transfer-file-local` extension provided in this sample. - -```kotlin -implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) -``` - -We also need to adjust the provider's `config.properties`. The property `edc.samples.transfer.01.asset.path` should -point to an existing file in our local environment, as this is the file that will be transferred. We also configure a -separate API context for the management API, like we learned in previous chapter. Then we add the property -`edc.dsp.callback.address`, which should point to our provider connector's DSP address. This is used as the callback -address during the contract negotiation. Since the DSP API is running on a different port (default is `8282`), we set -the webhook address to `http://localhost:8282/protocol` accordingly. - -### Consumer connector - -The consumer is the one "requesting" the data and providing a destination for it, i.e. a directory into which the -provider can copy the requested file. - -We configure the consumer's API ports in `consumer/config.properties`, so that it does not use the same ports as the -provider. In the config file, we also need to configure the API key authentication, as we're going to use -endpoints from the EDC's management API in this sample and integrated the extension for token-based API -authentication. Therefore, we add the property `edc.api.auth.key` and set it to e.g. `password`. And last, we also need -to configure the consumer's webhook address. We expose the DSP API endpoints on a different port and path than other -endpoints, so the property `edc.dsp.callback.address` is adjusted to match the DSP API port. - -## Run the sample - -Running this sample consists of multiple steps, that are executed one by one. - -### 1. Build and start the connectors - -The first step to running this sample is building and starting both the provider and the consumer connector. This is -done the same way as in the previous samples. - -```bash -./gradlew transfer:transfer-01-file-transfer:file-transfer-consumer:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-consumer/build/libs/consumer.jar -# in another terminal window: -./gradlew transfer:transfer-01-file-transfer:file-transfer-provider:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-provider/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-provider/build/libs/provider.jar -```` - -Assuming you didn't change the ports in config files, the consumer will listen on the ports `9191`, `9192` -(management API) and `9292` (PROTOCOL API) and the provider will listen on the ports `8181`, `8182` -(management API) and `8282` (PROTOCOL API). - -### 2. Initiate a contract negotiation - -In order to request any data, a contract agreement has to be negotiated between provider and consumer. The provider -offers all of their assets in the form of contract offers, which are the basis for such a negotiation. In the -`transfer-file-local` extension, we've added a contract definition (from which contract offers can be created) for the -file, but the consumer has yet to accept this offer. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also send counter offers in -addition to just confirming or declining an offer. - -In order to trigger the negotiation, we use a management API endpoint. We set our contract offer in the request -body. The contract offer is prepared in [contractoffer.json](contractoffer.json) and can be used as is. In a real -scenario, a potential consumer would first need to request a description of the provider's offers in order to get the -provider's contract offer. - -> Note, that we need to specify the `X-Api-Key` header, as we integrated token-based API authentication. The value -of the header has to match the value of the `edc.api.auth.key` property in the consumer's `config.properties`. - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" -``` - -In the response we'll get a UUID that we can use to get the contract agreement negotiated between provider and consumer. - -Sample output: - -```json -{"@id":"5a6b7e22-dc7d-4135-bc98-4cc5fd1dd1ed"} -``` - -### 3. Look up the contract agreement ID - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. This UUID is the ID of -the ongoing contract negotiation between consumer and provider. The negotiation sequence between provider and consumer -is executed asynchronously in the background by a state machine. Once both provider and consumer either reach the -`confirmed` or the `declined` state, the negotiation is finished. We can now use the UUID to check the current status -of the negotiation using an endpoint on the consumer side. Again, we use the `X-Api-Key` header with the same value -that's set in our consumer's `config.properties`. - -```bash -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" -``` - -This will return information about the negotiation, which contains e.g. the current state of the negotiation and, if the -negotiation has been completed successfully, the ID of a contract agreement. We can now use this agreement to request -the file. So we copy and store the agreement ID for the next step. - -Sample output: - -```json -{ - ... - "edc:contractAgreementId":"1:test-document:fb80be14-8e09-4e50-b65d-c269bc1f16d0", - "edc:state":"FINALIZED", - ... -} -``` - -If you see an output similar to the following, the negotiation has not yet been completed. In this case, -just wait for a moment and call the endpoint again. - -```json -{ - ... - "edc:state": "REQUESTED", - "edc:contractAgreementId": null, - ... -} -``` - -### 4. Request the file - -Now that we have a contract agreement, we can finally request the file. In the request body we need to specify -which asset we want transferred, the ID of the contract agreement, the address of the provider connector and where -we want the file transferred. The request body is prepared in [filetransfer.json](filetransfer.json). Before executing -the request, insert the contract agreement ID from the previous step and adjust the destination location for the file -transfer. Then run: - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" -``` - -Again, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and performed asynchronously. - -Sample output: - -```json -{"@id":"deeed974-8a43-4fd5-93ad-e1b8c26bfa44"} -``` - -Since transferring a file does not require any resource provisioning on either side, the transfer will be very quick and -most likely already done by the time you read the UUID. - ---- - -You can also check the logs of the connectors to see that the transfer has been completed: - -Consumer side: - -```bash -DEBUG 2022-05-03T10:37:59.599642754 Starting transfer for asset asset-id -DEBUG 2022-05-03T10:37:59.6071347 Transfer process initialised f925131b-d61e-48b9-aa15-0f5e2e749064 -DEBUG 2022-05-03T10:38:01.230902645 TransferProcessManager: Sending process f925131b-d61e-48b9-aa15-0f5e2e749064 request to http://localhost:8282/protocol -DEBUG 2022-05-03T10:38:01.260916372 Response received from connector. Status 200 -DEBUG 2022-05-03T10:38:01.285641788 TransferProcessManager: Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now REQUESTED -DEBUG 2022-05-03T10:38:06.246094874 Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now IN_PROGRESS -DEBUG 2022-05-03T10:38:06.246755642 Process f925131b-d61e-48b9-aa15-0f5e2e749064 is now COMPLETED -``` - -### 5. See transferred file - -After the file transfer is completed, we can check the destination path specified in the request for the file. Here, -we'll now find a file with the same content as the original file offered by the provider. - ---- - -[Next Chapter](../transfer-02-file-transfer-listener/README.md) diff --git a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts deleted file mode 100644 index 5d9ff687..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-consumer/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.api.observability) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) - -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties b/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties deleted file mode 100644 index 1655aa4f..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-consumer/config.properties +++ /dev/null @@ -1,11 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -web.http.protocol.port=9292 -web.http.protocol.path=/protocol - -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9292/protocol -edc.participant.id=consumer -edc.ids.id=urn:connector:consumer diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts b/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts deleted file mode 100644 index d1a7776c..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/build.gradle.kts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.api.client) - implementation(libs.edc.control.plane.api) - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.api.observability) - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - implementation(libs.edc.dsp) - - implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("provider.jar") -} diff --git a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties b/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties deleted file mode 100644 index 2383072c..00000000 --- a/transfer/transfer-01-file-transfer/file-transfer-provider/config.properties +++ /dev/null @@ -1,13 +0,0 @@ -web.http.port=8181 -web.http.path=/api -web.http.management.port=8182 -web.http.management.path=/management -web.http.protocol.port=8282 -web.http.protocol.path=/protocol -web.http.control.port=8283 -web.http.control.path=/control -edc.samples.transfer.01.asset.path=/path/to/file -edc.dsp.callback.address=http://localhost:8282/protocol -edc.participant.id=provider -edc.ids.id=urn:connector:provider -edc.control.endpoint=http://localhost:8283/control diff --git a/transfer/transfer-01-file-transfer/filetransfer.json b/transfer/transfer-01-file-transfer/filetransfer.json deleted file mode 100644 index 035bbb49..00000000 --- a/transfer/transfer-01-file-transfer/filetransfer.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "dataDestination": { - "type": "File", - "path": "{path to destination file}", - "keyName": "keyName" - }, - "protocol": "dataspace-protocol-http", - "assetId": "test-document", - "contractId": "{agreement ID}", - "connectorId": "provider", - "connectorAddress": "http://localhost:8282/protocol", - "privateProperties": {} -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts b/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts deleted file mode 100644 index 9051a9a9..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/build.gradle.kts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` - id("application") -} - -dependencies { - api(libs.edc.control.plane.spi) - api(libs.edc.data.plane.spi) - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.core) - implementation(libs.edc.data.plane.util) - implementation(libs.edc.data.plane.client) - implementation(libs.edc.data.plane.selector.client) - implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.transfer.data.plane) - implementation(libs.opentelemetry.annotations) -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java deleted file mode 100644 index 1fb0b4dd..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSink.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import io.opentelemetry.extension.annotations.WithSpan; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; - -import java.io.File; -import java.io.FileOutputStream; -import java.util.List; -import java.util.Objects; - -import static java.lang.String.format; -import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure.Reason.GENERAL_ERROR; - -class FileTransferDataSink extends ParallelSink { - private File file; - - @WithSpan - @Override - protected StreamResult transferParts(List parts) { - for (DataSource.Part part : parts) { - var fileName = part.name(); - try (var input = part.openStream()) { - try (var output = new FileOutputStream(file)) { - try { - input.transferTo(output); - } catch (Exception e) { - return getTransferResult(e, "Error transferring file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error creating file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error reading file %s", fileName); - } - } - return StreamResult.success(); - } - - private StreamResult getTransferResult(Exception e, String logMessage, Object... args) { - var message = format(logMessage, args); - monitor.severe(message, e); - return StreamResult.failure(new StreamFailure(List.of(message), GENERAL_ERROR)); - } - - public static class Builder extends ParallelSink.Builder { - - public static Builder newInstance() { - return new Builder(); - } - - public Builder file(File file) { - sink.file = file; - return this; - } - - @Override - protected void validate() { - Objects.requireNonNull(sink.file, "file"); - } - - private Builder() { - super(new FileTransferDataSink()); - } - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java deleted file mode 100644 index e7befc2a..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSinkFactory.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.util.concurrent.ExecutorService; - -class FileTransferDataSinkFactory implements DataSinkFactory { - private final Monitor monitor; - private final ExecutorService executorService; - private final int partitionSize; - - FileTransferDataSinkFactory(Monitor monitor, ExecutorService executorService, int partitionSize) { - this.monitor = monitor; - this.executorService = executorService; - this.partitionSize = partitionSize; - } - - @Override - public boolean canHandle(DataFlowRequest request) { - return "file".equalsIgnoreCase(request.getDestinationDataAddress().getType()); - } - - @Override - public DataSink createSink(DataFlowRequest request) { - var destination = request.getDestinationDataAddress(); - - // verify destination path - var path = destination.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - var destinationFile = new File(path.replaceAll("\\.", ".").replaceAll("/", "/")); - - return FileTransferDataSink.Builder.newInstance() - .file(destinationFile) - .requestId(request.getId()) - .partitionSize(partitionSize) - .executorService(executorService) - .monitor(monitor) - .build(); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - return Result.success(); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java deleted file mode 100644 index f946039f..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSource.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.spi.EdcException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.util.stream.Stream; - -class FileTransferDataSource implements DataSource { - - private final File file; - - FileTransferDataSource(File file) { - this.file = file; - } - - @Override - public StreamResult> openPartStream() { - var part = new Part() { - @Override - public String name() { - return file.getName(); - } - - @Override - public InputStream openStream() { - try { - return new FileInputStream(file); - } catch (FileNotFoundException e) { - throw new EdcException(e); - } - } - }; - return StreamResult.success(Stream.of(part)); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java deleted file mode 100644 index 3712b9ce..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferDataSourceFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; -import org.jetbrains.annotations.NotNull; - -import java.io.File; - -class FileTransferDataSourceFactory implements DataSourceFactory { - @Override - public boolean canHandle(DataFlowRequest dataRequest) { - return "file".equalsIgnoreCase(dataRequest.getSourceDataAddress().getType()); - } - - @Override - public DataSource createSource(DataFlowRequest request) { - var source = getFile(request); - return new FileTransferDataSource(source); - } - - @Override - public @NotNull Result validateRequest(DataFlowRequest request) { - var source = getFile(request); - if (!source.exists()) { - return Result.failure("Source file " + source.getName() + " does not exist!"); - } - - return Result.success(); - } - - @NotNull - private File getFile(DataFlowRequest request) { - var dataAddress = request.getSourceDataAddress(); - // verify source path - var sourceFileName = dataAddress.getStringProperty("filename"); - var path = dataAddress.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - sourceFileName = sourceFileName.replaceAll("\\.", ".").replaceAll("/", "/"); - path = path.replaceAll("\\.", ".").replaceAll("/", "/"); - return new File(path, sourceFileName); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java deleted file mode 100644 index 23d67e99..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/java/org/eclipse/edc/sample/extension/api/FileTransferExtension.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2021 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.api; - -import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; -import org.eclipse.edc.connector.contract.spi.types.offer.ContractDefinition; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.connector.policy.spi.PolicyDefinition; -import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.policy.model.PolicyType; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.asset.AssetIndex; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.asset.Asset; - -import java.nio.file.Path; - -import static org.eclipse.edc.spi.query.Criterion.criterion; - -public class FileTransferExtension implements ServiceExtension { - - public static final String USE_POLICY = "use-eu"; - private static final String EDC_ASSET_PATH = "edc.samples.transfer.01.asset.path"; - @Inject - private ContractDefinitionStore contractStore; - @Inject - private AssetIndex assetIndex; - @Inject - private PipelineService pipelineService; - @Inject - private DataTransferExecutorServiceContainer executorContainer; - @Inject - private PolicyDefinitionStore policyStore; - - @Override - public void initialize(ServiceExtensionContext context) { - var monitor = context.getMonitor(); - - var sourceFactory = new FileTransferDataSourceFactory(); - pipelineService.registerFactory(sourceFactory); - - var sinkFactory = new FileTransferDataSinkFactory(monitor, executorContainer.getExecutorService(), 5); - pipelineService.registerFactory(sinkFactory); - - var policy = createPolicy(); - policyStore.create(policy); - - registerDataEntries(context); - registerContractDefinition(policy.getUid()); - - context.getMonitor().info("File Transfer Extension initialized!"); - } - - private PolicyDefinition createPolicy() { - return PolicyDefinition.Builder.newInstance() - .id(USE_POLICY) - .policy(Policy.Builder.newInstance() - .type(PolicyType.SET) - .build()) - .build(); - } - - private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); - var assetPath = Path.of(assetPathSetting); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("path", assetPath.getParent().toString()) - .property("filename", assetPath.getFileName().toString()) - .build(); - - var assetId = "test-document"; - var asset = Asset.Builder.newInstance() - .id(assetId) - .dataAddress(dataAddress) - .build(); - - assetIndex.create(asset); - } - - private void registerContractDefinition(String uid) { - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(uid) - .contractPolicyId(uid) - .assetsSelectorCriterion(criterion(Asset.PROPERTY_ID, "=", "test-document")) - .build(); - - contractStore.save(contractDefinition); - } -} diff --git a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 5cc8c302..00000000 --- a/transfer/transfer-01-file-transfer/transfer-file-local/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.sample.extension.api.FileTransferExtension \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/README.md b/transfer/transfer-01-negotiation/README.md new file mode 100644 index 00000000..885d9a6e --- /dev/null +++ b/transfer/transfer-01-negotiation/README.md @@ -0,0 +1,242 @@ +# Implement a negotiation workflow between two connectors + + +Before two connectors can exchange actual data, negotiation has to take place. + +The final goal of this example is to showcase the negotiation workflow between two connectors so that +the actual data transfer can take place. The actual data transfer will be part of the next chapters. + +You will: + +* Create an asset on the provider (the asset will be the data to be shared) +* Create an access policy on the provider (the policy will define the access right to the data) +* Create a contract definition on the provider + +At this step, the consumer should be able to fetch the catalog from the provider and to see the +contract offer generated from the resources that have been created. + +Once the catalog is available, to access the data, the consumer should follow the following steps: + +* Perform a contract negotiation with the provider + +> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single +> asset. This will be deleted after the provider shutdown. + +The provider connector is the one managing assets and respective access policies for which a contract negotiation +can be executed while the consumer is the one "requesting" assets and initiating contract negotiations. + +## Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +The following steps assume your provider and consumer connector are still up and running. +If not, re-run them as described in the [Prerequisites](../transfer-00-prerequisites/README.md) chapter. + +### 1. Create an Asset on the provider side + +The provider connector needs to transfer a file to the location specified by the consumer connector +when the data are requested. In order to offer any data, the provider must maintain an internal list +of resources offered, through a contract offer, the so-called "catalog". + +The following [request](resources/create-asset.json) creates an asset on the provider connector. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ + -s | jq +``` + +> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for +> the purpose of this example. It will be the data that the consumer will pull on the sample +> execution. + +Additional properties on `HttpData` can be used to allow consumers to enrich the data request: + +- `proxyPath`: allows specifying additional path segments. +- `proxyQueryParams`: allows specifying query params. +- `proxyBody`: allows attaching a body. +- `proxyMethod`: allows specifying the Http Method (default `GET`) + +### 2. Create a Policy on the provider + +In order to manage the accessibility rules of an asset, it is essential to create a policy. However, +to keep things simple, we will choose a policy that gives direct access to all the assets that are +associated within the contract definitions. +This means that the consumer connector can request any asset from the provider connector. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ + -s | jq +``` + +### 3. Create a contract definition on Provider + +To ensure an exchange between providers and consumers, the supplier must create a contract offer for +the good, on the basis of which a contract agreement can be negotiated. The contract definition +associates policies to a selection of assets to generate the contract offers that will be put in the +catalog. In this case, the selection is empty, so every asset is attached to these policies + +```bash +curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ + -s | jq + +``` + +Sample output: + +```json +{ + ... + "@id": "1", + "edc:createdAt": 1674578184023, + ... +} +``` + +### 4. How to fetch catalog on consumer side + +In order to offer any data, the consumer can fetch the catalog from the provider, that will contain +all the contract offers available for negotiation. In our case, it will contain a single contract +offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following +request: + +```bash +curl -X POST "http://localhost:29193/management/v2/catalog/request" \ + -H 'Content-Type: application/json' \ + -d @transfer/transfer-01-negotiation/resources/fetch-catalog.json -s | jq +``` + +Sample output: + +```json +{ + "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", + "@type": "dcat:Catalog", + "dcat:dataset": { + "@id": "assetId", + "@type": "dcat:Dataset", + "odrl:hasPolicy": { + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "odrl:Set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "assetId" + }, + "dcat:distribution": [ + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpProxy" + }, + "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" + }, + { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpData" + }, + "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" + } + ], + "edc:name": "product description", + "edc:id": "assetId", + "edc:contenttype": "application/json" + }, + "dcat:service": { + "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", + "@type": "dcat:DataService", + "dct:terms": "connector", + "dct:endpointUrl": "http://localhost:19194/protocol" + }, + "edc:participantId": "anonymous", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` + +### 5. Negotiate a contract + +In order to request any data, a contract gets negotiated, and an agreement is resulting has to be +negotiated between providers and consumers. + +The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence +looks as follows: + +1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the + provider's offer!__) +2. Provider validates the received offer against its own offer +3. Provider either sends an agreement or a rejection, depending on the validation result +4. In case of successful validation, provider and consumer store the received agreement for later + reference + +Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also +send counter offers in addition to just confirming or declining an offer. + +```bash +curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ + -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ + -s | jq +``` + +Sample output: + +```json +{ + ... + "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", + "edc:createdAt": 1685525281848, + ... +} +``` + +### 6. Getting the contract agreement id + +After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. +This UUID is the ID of the ongoing contract negotiation between consumer and provider. The +negotiation sequence between provider and consumer is executed asynchronously in the background by a +state machine. Once both provider and consumer either reach the `confirmed` or the `declined` +state, the negotiation is finished. We can now use the UUID to check the current status of the +negotiation using an endpoint on the consumer side. + +```bash +curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ + --header 'Content-Type: application/json' \ + -s | jq +``` + +Sample output: + +```json +{ + "@type": "edc:ContractNegotiationDto", + "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", + "edc:type": "CONSUMER", + "edc:protocol": "dataspace-protocol-http", + "edc:state": "FINALIZED", + "edc:counterPartyAddress": "http://localhost:19194/protocol", + "edc:callbackAddresses": [], + "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", <--------- + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` +At this point contract negotiation has been successfully completed. +The connectors are now ready to enter the data transfer phase. + +Note down the `contractAgreementId`. You will need it in the next chapters. + +[Next Chapter](../transfer-02-consumer-pull/README.md) \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-asset.json b/transfer/transfer-01-negotiation/resources/create-asset.json new file mode 100644 index 00000000..b6abaf3f --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-asset.json @@ -0,0 +1,18 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "asset": { + "@id": "assetId", + "properties": { + "name": "product description", + "contenttype": "application/json" + } + }, + "dataAddress": { + "type": "HttpData", + "name": "Test asset", + "baseUrl": "https://jsonplaceholder.typicode.com/users", + "proxyPath": "true" + } +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-contract-definition.json b/transfer/transfer-01-negotiation/resources/create-contract-definition.json new file mode 100644 index 00000000..4094d276 --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-contract-definition.json @@ -0,0 +1,9 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "1", + "accessPolicyId": "aPolicy", + "contractPolicyId": "aPolicy", + "assetsSelector": [] +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/create-policy.json b/transfer/transfer-01-negotiation/resources/create-policy.json new file mode 100644 index 00000000..e62eb3cc --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/create-policy.json @@ -0,0 +1,13 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "aPolicy", + "policy": { + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [] + } +} \ No newline at end of file diff --git a/transfer/transfer-01-negotiation/resources/fetch-catalog.json b/transfer/transfer-01-negotiation/resources/fetch-catalog.json new file mode 100644 index 00000000..74994a15 --- /dev/null +++ b/transfer/transfer-01-negotiation/resources/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "providerUrl": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-01-negotiation/resources/negotiate-contract.json similarity index 57% rename from transfer/transfer-04-open-telemetry/contractoffer.json rename to transfer/transfer-01-negotiation/resources/negotiate-contract.json index 5baecd25..33a46d2a 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-01-negotiation/resources/negotiate-contract.json @@ -5,20 +5,20 @@ }, "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", "consumerId": "consumer", "providerId": "provider", - "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", + "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "assetId": "assetId", "policy": { - "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", - "@type": "set", + "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", + "@type": "Set", "odrl:permission": [], "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "test-document" + "odrl:target": "assetId" } } -} +} \ No newline at end of file diff --git a/transfer/transfer-02-consumer-pull/README.md b/transfer/transfer-02-consumer-pull/README.md new file mode 100644 index 00000000..c1e41227 --- /dev/null +++ b/transfer/transfer-02-consumer-pull/README.md @@ -0,0 +1,131 @@ +# Implement a simple "Consumer Pull" Http transfer flow + +The purpose of this sample is to show a data exchange between 2 connectors, one representing the +data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find +more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +This sample consists of the following steps: + +* Perform a file transfer initiated by the consumer +* The provider will send an EndpointDataReference to the consumer +* The consumer will call the endpoint and fetch the data + +## Prerequisites + +The following steps assume your provider and consumer connectors are still up and running and contract +negotiation has taken place successfully. +If not, re-visit the [Prerequisites](../transfer-00-prerequisites/README.md) +and [Negotiation](../transfer-01-negotiation/README.md) chapters. + +# Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 1. Start a http server + +As a pre-requisite, you need to have a http server that runs on port 4000 and logs all the incoming requests, it will +be mandatory to get the EndpointDataReference that will be used to get the data. + +```bash +docker build -t http-request-logger util/http-request-logger +docker run -p 4000:4000 http-request-logger +``` + +### 2. Start the transfer + +In the [request body](resources/start-transfer.json), we need to specify which asset we want transferred, the ID of the contract agreement, the address of the +provider connector and where we want the file transferred. +Before executing the request, insert the `contractAgreementId` from the previous chapter. Then run: + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-02-consumer-pull/resources/start-transfer.json \ + -s | jq + +``` + +> the "HttpProxy" method is used for the consumer pull method, and it means that it will be up to +> the consumer to request the data to the provider and that the request will be a proxy for the +> datasource + +Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( +process id) created on the consumer +side, because like the contract negotiation, the data transfer is handled in a state machine and +performed asynchronously. + +Sample output: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:createdAt": 1674078357807, + ... +} +``` + +### 3. Check the transfer status + +Due to the nature of the transfer, it will be very fast and most likely already done by the time you +read the UUID. + +```bash +curl http://localhost:29193/management/v2/transferprocesses/ +``` + +You should see the Transfer Process in `STARTED` state: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:state": "STARTED", + ... +} + +``` + +> Note that for the consumer pull scenario the TP will stay in STARTED state after the data has been transferred successfully. +> It might get eventually get shifted to TERMINATED or DEPROVISIONED by other resources, but this is not scope of this sample. + +### 4. Check the data + +At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed +to get the data from the provider: + +```json +{ + "id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "endpoint": "http://localhost:29291/public/", + "authKey": "Authorization", + "authCode": "eyJhbGciOiJSUzI1NiJ9.eyJkYWQiOiJ7XCJwcm9wZXJ0aWVzXCI6e1wiYXV0aEtleVwiOlwiQXV0aG9yaXphdGlvblwiLFwiYmFzZVVybFwiOlwiaHR0cDpcL1wvbG9jYWxob3N0OjE5MjkxXC9wdWJsaWNcL1wiLFwiYXV0aENvZGVcIjpcImV5SmhiR2NpT2lKU1V6STFOaUo5LmV5SmtZV1FpT2lKN1hDSndjbTl3WlhKMGFXVnpYQ0k2ZTF3aVltRnpaVlZ5YkZ3aU9sd2lhSFIwY0hNNlhDOWNMMnB6YjI1d2JHRmpaV2h2YkdSbGNpNTBlWEJwWTI5a1pTNWpiMjFjTDNWelpYSnpYQ0lzWENKdVlXMWxYQ0k2WENKVVpYTjBJR0Z6YzJWMFhDSXNYQ0owZVhCbFhDSTZYQ0pJZEhSd1JHRjBZVndpZlgwaUxDSmxlSEFpT2pFMk56UTFPRGcwTWprc0ltTnBaQ0k2SWpFNk1XVTBOemc1TldZdE9UQXlOUzAwT1dVeExUazNNV1F0WldJNE5qVmpNemhrTlRRd0luMC5ITFJ6SFBkT2IxTVdWeWdYZi15a0NEMHZkU3NwUXlMclFOelFZckw5eU1tQjBzQThwMHFGYWV0ZjBYZHNHMG1HOFFNNUl5NlFtNVU3QnJFOUwxSE5UMktoaHFJZ1U2d3JuMVhGVUhtOERyb2dSemxuUkRlTU9ZMXowcDB6T2MwNGNDeFJWOEZoemo4UnVRVXVFODYwUzhqbU4wZk5sZHZWNlFpUVFYdy00QmRTQjNGYWJ1TmFUcFh6bDU1QV9SR2hNUGphS2w3RGsycXpJZ0ozMkhIdGIyQzhhZGJCY1pmRk12aEM2anZ2U1FieTRlZXU0OU1hclEydElJVmFRS1B4ajhYVnI3ZFFkYV95MUE4anNpekNjeWxyU3ljRklYRUV3eHh6Rm5XWmczV2htSUxPUFJmTzhna2RtemlnaXRlRjVEcmhnNjZJZzJPR0Eza2dBTUxtc3dcIixcInByb3h5TWV0aG9kXCI6XCJ0cnVlXCIsXCJwcm94eVF1ZXJ5UGFyYW1zXCI6XCJ0cnVlXCIsXCJwcm94eUJvZHlcIjpcInRydWVcIixcInR5cGVcIjpcIkh0dHBEYXRhXCIsXCJwcm94eVBhdGhcIjpcInRydWVcIn19IiwiZXhwIjoxNjc0NTg4NDI5LCJjaWQiOiIxOjFlNDc4OTVmLTkwMjUtNDllMS05NzFkLWViODY1YzM4ZDU0MCJ9.WhbTzERmM75mNMUG2Sh-8ZW6uDQCus_5uJPvGjAX16Ucc-2rDcOhAxrHjR_AAV4zWjKBHxQhYk2o9jD-9OiYb8Urv8vN4WtYFhxJ09A0V2c6lB1ouuPyCA_qKqJEWryTbturht4vf7W72P37ERo_HwlObOuJMq9CS4swA0GBqWupZHAnF-uPIQckaS9vLybJ-gqEhGxSnY4QAZ9-iwSUhkrH8zY2GCDkzAWIPmvtvRhAs9NqVkoUswG-ez1SUw5bKF0hn2OXv_KhfR8VsKKYUbKDQf5Wagk7rumlYbXMPNAEEagI4R0xiwKWVTfwwZPy_pYnHE7b4GQECz3NjhgdIw", + "properties": { + "cid": "1:1e47895f-9025-49e1-971d-eb865c38d540" + } +} +``` + +Once this json is read, use a tool like postman or curl to execute the following query, to read the +data + +```bash +curl --location --request GET 'http://localhost:29291/public/' --header 'Authorization: ' +``` + +At the end, and to be sure that you correctly achieved the pull, you can check if the data you get +is the same as the one you can get at https://jsonplaceholder.typicode.com/users + + +Since we configured the `HttpData` with `proxyPath`, we could also ask for a specific user with: + +```bash +curl --location --request GET 'http://localhost:29291/public/1' --header 'Authorization: ' +``` + +And the data returned will be the same as in https://jsonplaceholder.typicode.com/users/1 + +Your first data transfer has been completed successfully. +Continue with the [next chapter](../transfer-03-provider-push/README.md) to run through a "provider push" scenario. \ No newline at end of file diff --git a/transfer/transfer-02-consumer-pull/resources/start-transfer.json b/transfer/transfer-02-consumer-pull/resources/start-transfer.json new file mode 100644 index 00000000..e28fc9c5 --- /dev/null +++ b/transfer/transfer-02-consumer-pull/resources/start-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} \ No newline at end of file diff --git a/transfer/transfer-02-file-transfer-listener/README.md b/transfer/transfer-02-file-transfer-listener/README.md deleted file mode 100644 index 0bc11666..00000000 --- a/transfer/transfer-02-file-transfer-listener/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Implement a simple transfer listener - -In this sample, we build upon the [file transfer sample](../transfer-01-file-transfer/README.md) to add functionality -to react to transfer completion on the consumer connector side. - -We will use the provider from the [file transfer sample](../transfer-01-file-transfer/README.md), and the consumer -built on the consumer from that sample, with a transfer process listener added. - -Also, in order to keep things organized, the code in this example has been separated into several Java modules: - -- `file-transfer-listener-consumer`: this is where the extension definition and dependencies reside for the consumer connector -- `listener`: contains the `TransferProcessListener` implementation - -## Create the listener - -A TransferProcessListener may define methods that are invoked after a transfer changes state, for example, to notify an -external application on the consumer side after data has been produced (i.e. the transfer moves to the completed state). - -```java -// in TransferListenerExtension.java - @Override - public void initialize(ServiceExtensionContext context) { - // ... - var transferProcessObservable = context.getService(TransferProcessObservable.class); - transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); - } -``` - -```java -public class MarkerFileCreator implements TransferProcessListener { - - /** - * Callback invoked by the EDC framework when a transfer has completed. - * - * @param process - */ - @Override - public void completed(final TransferProcess process) { - // ... - } -} -``` - -## Perform a file transfer - -Let's rebuild and run them both: - -```bash -./gradlew transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer:build -java -Dedc.fs.config=transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties -jar transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build/libs/consumer.jar -# in another terminal window: -./gradlew transfer:transfer-01-file-transfer:file-transfer-provider:build -java -Dedc.fs.config=transfer/transfer-01-file-transfer/file-transfer-provider/config.properties -jar transfer/transfer-01-file-transfer/file-transfer-provider/build/libs/provider.jar -```` - -Assuming you didn't change the config files, the consumer will expose management api on port `9192` and the custom -api endpoints on port `9191` and the provider will listen on port `8181`. -Open another terminal window (or any REST client of your choice) and execute the following REST requests like in the -previous sample: - -```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}" -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" -``` - -> **Replace `{UUID}` in the second request with the UUID received as the response to the first request!** -> -> **In `transfer/transfer-01-file-transfer/filetransfer.json`: Copy the contract agreement's ID from the second response, -> substitute it for `{agreement ID}` in the last request JSON body and adjust the `dataDestination.properties.path` to match your local dev machine!** - -- `curl` will return the ID of the transfer process on the consumer connector. - -The consumer should spew out logs similar to: - -```bash -DEBUG 2022-04-14T16:23:13.4042547 Starting transfer for asset test-document -DEBUG 2022-04-14T16:23:13.4072776 Transfer process initialised 6804ed96-298e-4992-b72d-2366d97cf7a6 -DEBUG 2022-04-14T16:23:13.8801678 TransferProcessManager: Sending process 6804ed96-298e-4992-b72d-2366d97cf7a6 request to http://localhost:8282/protocol -DEBUG 2022-04-14T16:23:13.9341802 TransferProcessManager: Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now REQUESTED -DEBUG 2022-04-14T16:23:18.9048494 Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now IN_PROGRESS -DEBUG 2022-04-14T16:23:18.9048494 Process 6804ed96-298e-4992-b72d-2366d97cf7a6 is now COMPLETED -INFO 2022-04-14T16:23:18.9048494 Transfer Listener successfully wrote file C:\Users\pechande\dev\coding\EDC\marker.txt -``` - -Then check `/path/on/yourmachine`, which should now contain a file named `marker.txt` in addition to the file defined -in `dataDestination.properties.path` in `transfer/transfer-01-file-transfer/filetransfer.json`. - ---- - -[Previous Chapter](../transfer-01-file-transfer/README.md) | [Next Chapter](../transfer-03-modify-transferprocess/README.md) diff --git a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts b/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts deleted file mode 100644 index e8e9e82c..00000000 --- a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/build.gradle.kts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) - - implementation(libs.edc.http) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) - - implementation(project(":transfer:transfer-02-file-transfer-listener:listener")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - exclude("**/pom.properties", "**/pom.xml") - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties b/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties deleted file mode 100644 index e162bff0..00000000 --- a/transfer/transfer-02-file-transfer-listener/file-transfer-listener-consumer/config.properties +++ /dev/null @@ -1,9 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -web.http.protocol.port=9292 -web.http.protocol.path=/protocol - -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9292/protocol diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 97c66199..00000000 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.sample.extension.listener.TransferListenerExtension \ No newline at end of file diff --git a/transfer/transfer-03-modify-transferprocess/README.md b/transfer/transfer-03-modify-transferprocess/README.md deleted file mode 100644 index 1f1c9845..00000000 --- a/transfer/transfer-03-modify-transferprocess/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Modify a TransferProcess - -In the last transfer samples (`transfer-01` and `transfer-02`) we saw how data can be transferred easily, what a -`TransferProcess` is and how to react to it easily through the listener. This sample will show how `TransferProcess` -objects can be modified externally in a thread-safe and consistent way. - -## Problem statement - -The `TransferProcessManager` (TPM), which is the central state machine handling transfer processes, follows this basic -operational pattern: - -1. take transfer process (TP) out of `TransferProcessStore` (TPS) -2. take appropriate action, e.g. provision or deprovision resources -3. update state of TP -4. put back into TPS - -All those steps happen in a non-atomic way, so when a TP currently processed by the TPM is modified on another thread, -there is a strong possibility that that change will get overwritten or worse, may cause the state machine to be in an -illegal state. - -A common pattern would be some sort of watchdog, where TPs that have not advanced their state in a given amount of time -are automatically cancelled or errored out. The following code snippet shows a typical TPM state transition: - -```java -// get out of store - var tpList = store.nextForState(IN_PROGRESS.code(),batchSize); -// take appropriate action, e.g. check if complete -var statusChecker=...; -foreach(var tp:tpList){ - if(statusChecker.isComplete()){ - //update state - tp.transitionComplete(); - // put back into TPS - store.update(tp); - } -} -``` -and then consider a watchdog that runs on another thread and fires every X minutes -```java - -private void watchDog(){ - var longRunningTpList = store.nextForState(IN_PROGRESS.code(), 5); - // filter list based on last state update - var longRunningTpList = /*filter expression*/; - for(var tp : longRunningTpList){ - tp.transitionError("timeout"); - store.update(tp); - } -} -``` - -Now the problem becomes apparent when the `watchDog()` fires exactly here: -```java -//... - if(statusChecker.isComplete()){ - - // |<-- watchDog() fires here! - - //update state - tp.transitionComplete(); - // ... - } -``` - -then the TP would first go to the `ERROR` state, but then immediately to the `COMPLETED` state, because the TPM and the watchdog -have different object references to the same TP. We essentially have a race condition at our hands, resulting in the TP never -"erroring out". - -## About this sample -Please note that this sample does _not actually transfer anything_, it merely shows how to modify a transfer process -outside the main state machine. - -Modules: -- `simulator`: used to insert a dummy transfer process, that never completes to simulate the use of a watchdog -- `watchdog`: spins up a periodic task that checks for timed-out TPs and sets them to `ERROR` -- `consumer`: the build configuration - -In order to run the sample, enter the following commands in a shell: - -```bash -./gradlew transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer:build -java -Dedc.fs.config=transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties -jar transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build/libs/consumer.jar -``` - ---- - -[Previous Chapter](../transfer-02-file-transfer-listener/README.md) | [Next Chapter](../transfer-04-open-telemetry/README.md) diff --git a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts deleted file mode 100644 index 497de949..00000000 --- a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/build.gradle.kts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * ZF Friedrichshafen AG - add dependency - * - */ - -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(project(":transfer:transfer-03-modify-transferprocess:watchdog")) - implementation(project(":transfer:transfer-03-modify-transferprocess:simulator")) - - implementation(libs.edc.control.plane.core) - implementation(libs.edc.http) - - implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - - implementation(libs.edc.auth.tokenbased) - implementation(libs.edc.management.api) - - implementation(libs.edc.dsp) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("consumer.jar") -} diff --git a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties b/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties deleted file mode 100644 index 863cdab3..00000000 --- a/transfer/transfer-03-modify-transferprocess/modify-transferprocess-consumer/config.properties +++ /dev/null @@ -1,6 +0,0 @@ -web.http.port=9191 -web.http.path=/api -web.http.management.port=9192 -web.http.management.path=/management -edc.api.auth.key=password -edc.dsp.callback.address=http://localhost:9191/protocol diff --git a/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts deleted file mode 100644 index 66cd3278..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/build.gradle.kts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` -} - -dependencies { - implementation(libs.edc.control.plane.core) -} diff --git a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java b/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java deleted file mode 100644 index eb071a9d..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/src/main/java/org/eclipse/edc/sample/extension/transfer/TransferSimulationExtension.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; -import org.eclipse.edc.connector.transfer.spi.types.ProvisionedDataDestinationResource; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.jetbrains.annotations.NotNull; - -import java.time.Clock; -import java.time.Duration; -import java.util.Timer; -import java.util.TimerTask; - -public class TransferSimulationExtension implements ServiceExtension { - - public static final String TEST_TYPE = "test-type"; - private static final long ALMOST_TEN_MINUTES = Duration.ofMinutes(9).plusSeconds(55).toMillis(); - @Inject - private TransferProcessStore store; - - @Inject - private Clock clock; - - @Override - public void initialize(ServiceExtensionContext context) { - //Insert a test TP after a delay to simulate a zombie transfer - new Timer().schedule( - new TimerTask() { - @Override - public void run() { - var tp = TransferProcess.Builder.newInstance() - .id("tp-sample-transfer-03") - .dataRequest(getRequest()) - .state(TransferProcessStates.STARTED.code()) - .stateTimestamp(clock.millis() - ALMOST_TEN_MINUTES) - .build(); - tp.addProvisionedResource(createDummyResource()); - - context.getMonitor().info("Insert Dummy TransferProcess"); - store.save(tp); - } - }, - 5000 - ); - } - - @NotNull - private ProvisionedDataDestinationResource createDummyResource() { - return new ProvisionedDataDestinationResource() { - }; - } - - private DataRequest getRequest() { - return DataRequest.Builder.newInstance() - .id("sample-transfer-03-datarequest") - .assetId("assetId") - .contractId("contractId") - .destinationType(TEST_TYPE) - .connectorAddress("http//localhost:9999") - .protocol("dataspace-protocol-http") - .build(); - } - -} diff --git a/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index f8129d56..00000000 --- a/transfer/transfer-03-modify-transferprocess/simulator/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2020, 2021 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.sample.extension.transfer.TransferSimulationExtension - diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts b/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts deleted file mode 100644 index c472d4ab..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/build.gradle.kts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ - -plugins { - `java-library` -} - -dependencies { - implementation(libs.edc.control.plane.core) - implementation(libs.edc.data.plane.selector.core) -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java deleted file mode 100644 index 380e28e3..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/Watchdog.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.watchdog; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.query.Criterion; - -import java.time.Clock; -import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static java.lang.String.format; -import static java.time.Instant.ofEpochMilli; -import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; - -public class Watchdog { - - private final Monitor monitor; - private final TransferProcessStore store; - private final Clock clock; - private ScheduledExecutorService executor; - - public Watchdog(Monitor monitor, TransferProcessStore store, Clock clock) { - this.monitor = monitor; - this.store = store; - this.clock = clock; - } - - public void start() { - executor = Executors.newSingleThreadScheduledExecutor(); - // run every 10 minutes, no initial delay - executor.scheduleAtFixedRate(this::check, 10, 10, TimeUnit.SECONDS); - } - - public void stop() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private void check() { - monitor.info("Running watchdog"); - - var states = store.nextNotLeased(3, new Criterion("state", "=", STARTED.code())); - states.stream().filter(tp -> isExpired(tp.getStateTimestamp(), Duration.ofSeconds(10))) - .forEach(tp -> { - monitor.info(format("will retire TP with id [%s] due to timeout", tp.getId())); - - tp.transitionTerminating("timeout by watchdog"); - store.save(tp); - }); - } - - private boolean isExpired(long stateTimestamp, Duration maxAge) { - return ofEpochMilli(stateTimestamp).isBefore(clock.instant().minus(maxAge)); - } -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java deleted file mode 100644 index 3c564f14..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/java/org/eclipse/edc/sample/extension/watchdog/WatchdogExtension.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - Initial implementation - * - */ - -package org.eclipse.edc.sample.extension.watchdog; - -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; - -import java.time.Clock; - -public class WatchdogExtension implements ServiceExtension { - - @Inject - private TransferProcessStore store; - - @Inject - private Clock clock; - - private Watchdog watchDog; - - @Override - public void initialize(ServiceExtensionContext context) { - watchDog = new Watchdog(context.getMonitor(), store, clock); - } - - @Override - public void start() { - watchDog.start(); - } - - @Override - public void shutdown() { - watchDog.stop(); - } -} diff --git a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 72409f33..00000000 --- a/transfer/transfer-03-modify-transferprocess/watchdog/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2020, 2021 Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Contributors: -# Microsoft Corporation - initial API and implementation -# -# - -org.eclipse.edc.sample.extension.watchdog.WatchdogExtension - diff --git a/transfer/transfer-03-provider-push/README.md b/transfer/transfer-03-provider-push/README.md new file mode 100644 index 00000000..07277efa --- /dev/null +++ b/transfer/transfer-03-provider-push/README.md @@ -0,0 +1,69 @@ +# Implement a simple "Provider Push" Http transfer flow + +This sample demonstrates the "provider push" use case that you can find more details +on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). + +This samples consists of: + +* Performing a file transfer + * The consumer will initiate a file transfer + * Provider Control Plane retrieves the DataAddress of the actual data source and creates a + DataFlowRequest based on the received DataRequest and this data address +* Provider Data Plane fetches data from the actual data source +* Provider Data Plane pushes data to the consumer service + +## Prerequisites + +The following steps assume your provider and consumer connectors are still up and running and contract +negotiation has taken place successfully. Furthermore, the http server should be up as well. +If not, re-visit the [Prerequisites](../transfer-00-prerequisites/README.md) +, [Negotiation](../transfer-01-negotiation/README.md) and [Consumer Pull](../transfer-02-consumer-pull/README.md) chapters. + +# Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### 1. Start the transfer + +Before executing the request, modify the [request body](resources/start-transfer.json) by inserting the contract agreement ID +from the [Negotiation](../transfer-01-negotiation/README.md) chapter. +You can re-use the same asset, policies and contract negotiation from before. + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-03-provider-push/resources/start-transfer.json \ + -s | jq +``` +> keep in mind that, to make a transfer with a provider push method, the dataDestination type should +> be any value different from the "HttpProxy". + +Sample output: + +```json + { + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "edc:createdAt": 1674078357807, + ... +} +``` + +### 2. Check the transfer status + +Due to the nature of the transfer, it will be very fast and most likely already done by the time you +read the UUID. + +```bash +curl http://localhost:29193/management/v2/transferprocesses/ +``` + +Notice the transfer COMPLETED state + +### 3. Check the data + +At this step, you can check the data by checking the log of the http server exposed on port 4000, you should see a log +that shows the same data that you can get from https://jsonplaceholder.typicode.com/users. + +[Next Chapter](../transfer-04-event-consumer/README.md) \ No newline at end of file diff --git a/transfer/transfer-03-provider-push/resources/start-transfer.json b/transfer/transfer-03-provider-push/resources/start-transfer.json new file mode 100644 index 00000000..c4636720 --- /dev/null +++ b/transfer/transfer-03-provider-push/resources/start-transfer.json @@ -0,0 +1,15 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "connectorAddress": "http://localhost:19194/protocol", + "contractId": "", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpData", + "baseUrl": "http://localhost:4000/api/consumer/store" + } +} \ No newline at end of file diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md new file mode 100644 index 00000000..85952ce9 --- /dev/null +++ b/transfer/transfer-04-event-consumer/README.md @@ -0,0 +1,112 @@ +# Implement a simple event consumer + +In this sample, we build upon the [Consumer Pull](../transfer-02-consumer-pull/README.md) chapter to add functionality +to react to transfer completion on the consumer connector side. + +Also, in order to keep things organized, the code in this example has been separated into several Java modules: + +- `consumer-with-listener`: the consumer connector which will be extended by the event consumer +- `listener`: contains the `TransferProcessListener` implementation which will consume an event + +## Inspect the listener + +A `TransferProcessListener` may define methods that are invoked after a transfer changes state, for example, to notify an +external application on the consumer side after data has been produced (i.e. the transfer moves to the completed state). + +```java +// in TransferListenerExtension.java + @Override + public void initialize(ServiceExtensionContext context) { + // ... + var transferProcessObservable = context.getService(TransferProcessObservable.class); + transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); + } +``` + +The `TransferProcessStartedListener` implements the `TransferProcessListener` interface. +It will consume the transfer `STARTED` event and write a log message. + +```java +public class TransferProcessStartedListener implements TransferProcessListener { + + private final Monitor monitor; + + public TransferProcessStartedListener(Monitor monitor) { + this.monitor = monitor; + } + + /** + * Callback invoked by the EDC framework when a transfer is about to be completed. + * + * @param process the transfer process that is about to be completed. + */ + @Override + public void preStarted(final TransferProcess process) { + monitor.debug("TransferProcessStartedListener received STARTED event"); + // do something meaningful before transfer start + } +} +``` + +## Run the sample + +Assuming your provider connector is still running, we can re-use the existing assets and contract definitions stored on +provider side. If not, set up your assets and contract definitions as described in the [Negotiation](../transfer-01-negotiation/README.md) +chapter. + +### 1. Build & launch the consumer with listener extension + +This consumer connector is based on a different build file, hence a new jar file will be produced. +Make sure to terminate your current consumer connector from the previous chapters. +That way we unblock the ports and can reuse the known configuration files and API calls. + +Run this to build and launch the consumer with listener extension: + +```bash +./gradlew transfer:transfer-04-event-consumer:consumer-with-listener:build +java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-00-prerequisites/resources/configuration/consumer-vault.properties -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar +```` + +### 2. Negotiate a new contract + +```bash +curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ + -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ + -s | jq +``` + +### 3. Get the contract agreement id + +```bash +curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ + --header 'Content-Type: application/json' \ + -s | jq +``` + +### 4. Perform a file transfer + +Replace the `contractId` property inside the [request body](../transfer-02-consumer-pull/resources/start-transfer.json) with the contract agreement id from the previous call. +Afterward run: + +```bash +curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-02-consumer-pull/resources/start-transfer.json \ + -s | jq +``` + +### 5. Inspect the logs + +The consumer should spew out logs similar to: + +```bash +DEBUG 2023-10-16T09:29:45.316908 [TransferProcessManagerImpl] TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8fa81 is now in state REQUESTED +DEBUG 2023-10-16T09:29:46.269998 DSP: Incoming TransferStartMessage for class org.eclipse.edc.connector.transfer.spi.types.TransferProcess process: 762b5a0c-43fb-4b8b-8022-669043c8fa81 +DEBUG 2023-10-16T09:29:46.271592 TransferProcessStartedListener received STARTED event <---------------------------- +DEBUG 2023-10-16T09:29:46.27174 TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8fa81 is now in state STARTED +``` + +If you see the `TransferProcessStartedListener received STARTED event` log message, it means that your event consumer has been +configured successfully. + +[Next Chapter](../transfer-05-open-telemetry/README.md) diff --git a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts b/transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts similarity index 86% rename from transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts rename to transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts index 86bc1e6b..805fe5f3 100644 --- a/transfer/transfer-07-provider-push-http/http-push-connector/build.gradle.kts +++ b/transfer/transfer-04-event-consumer/consumer-with-listener/build.gradle.kts @@ -18,11 +18,9 @@ plugins { alias(libs.plugins.shadow) } -repositories { - mavenCentral() -} - dependencies { + + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.control.plane.api.client) implementation(libs.edc.dsp) @@ -31,6 +29,7 @@ dependencies { implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.data.plane.selector.api) implementation(libs.edc.data.plane.selector.core) @@ -39,6 +38,8 @@ dependencies { implementation(libs.edc.data.plane.api) implementation(libs.edc.data.plane.core) implementation(libs.edc.data.plane.http) + + implementation(project(":transfer:transfer-04-event-consumer:listener")) } application { @@ -50,6 +51,6 @@ var distZip = tasks.getByName("distZip") tasks.withType { mergeServiceFiles() - archiveFileName.set("push-connector.jar") + archiveFileName.set("connector.jar") dependsOn(distTar, distZip) } diff --git a/transfer/transfer-02-file-transfer-listener/listener/build.gradle.kts b/transfer/transfer-04-event-consumer/listener/build.gradle.kts similarity index 100% rename from transfer/transfer-02-file-transfer-listener/listener/build.gradle.kts rename to transfer/transfer-04-event-consumer/listener/build.gradle.kts diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java similarity index 52% rename from transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java rename to transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java index 4f2714f4..515a9e26 100644 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/MarkerFileCreator.java +++ b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListener.java @@ -18,17 +18,11 @@ import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.monitor.Monitor; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import static java.lang.String.format; - -public class MarkerFileCreator implements TransferProcessListener { +public class TransferProcessStartedListener implements TransferProcessListener { private final Monitor monitor; - public MarkerFileCreator(Monitor monitor) { + public TransferProcessStartedListener(Monitor monitor) { this.monitor = monitor; } @@ -38,18 +32,8 @@ public MarkerFileCreator(Monitor monitor) { * @param process the transfer process that is about to be completed. */ @Override - public void preCompleted(final TransferProcess process) { - Path path = Path.of(process.getDataRequest().getDataDestination().getStringProperty("path")); - if (!Files.isDirectory(path)) { - path = path.getParent(); - } - path = path.resolve("marker.txt"); - - try { - Files.writeString(path, "Transfer complete"); - monitor.info(format("Transfer Listener successfully wrote file %s", path)); - } catch (IOException e) { - monitor.warning(format("Could not write file %s", path), e); - } + public void preStarted(final TransferProcess process) { + monitor.debug("TransferProcessStartedListener received STARTED event"); + // do something meaningful before transfer start } } diff --git a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java similarity index 82% rename from transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java rename to transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java index df2cf070..a2e5f47e 100644 --- a/transfer/transfer-02-file-transfer-listener/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferListenerExtension.java +++ b/transfer/transfer-04-event-consumer/listener/src/main/java/org/eclipse/edc/sample/extension/listener/TransferProcessStartedListenerExtension.java @@ -18,14 +18,13 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; -public class TransferListenerExtension implements ServiceExtension { +public class TransferProcessStartedListenerExtension implements ServiceExtension { @Override public void initialize(ServiceExtensionContext context) { var transferProcessObservable = context.getService(TransferProcessObservable.class); var monitor = context.getMonitor(); - - transferProcessObservable.registerListener(new MarkerFileCreator(monitor)); + transferProcessObservable.registerListener(new TransferProcessStartedListener(monitor)); } } diff --git a/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..b53afdfc --- /dev/null +++ b/transfer/transfer-04-event-consumer/listener/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.extension.listener.TransferProcessStartedListenerExtension \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml deleted file mode 100644 index b3ad0fca..00000000 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ /dev/null @@ -1,78 +0,0 @@ -version: "3.8" - -services: - - consumer: - build: - context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile - volumes: - - ./:/resources - ports: - - "9191:9191" - - "9192:9192" - environment: - EDC_HOSTNAME: consumer - OTEL_SERVICE_NAME: consumer - OTEL_TRACES_EXPORTER: jaeger - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 - OTEL_METRICS_EXPORTER: prometheus - WEB_HTTP_PORT: 9191 - WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 9192 - WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_PROTOCOL_PORT: 9292 - WEB_HTTP_PROTOCOL_PATH: /protocol - WEB_HTTP_CONTROL_PORT: 9193 - WEB_HTTP_CONTROL_PATH: /control - EDC_CONTROL_ENDPOINT: http://consumer:9193/control - EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol - EDC_PARTICIPANT_ID: consumer - EDC_API_AUTH_KEY: password - entrypoint: java - -javaagent:/app/libs/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/resources/logging.properties - -jar /app/connector.jar - - provider: - build: - context: ../.. - dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile - volumes: - - ./:/resources - ports: - - "8181:8181" - - "8182:8182" - environment: - EDC_HOSTNAME: provider - OTEL_SERVICE_NAME: provider - OTEL_TRACES_EXPORTER: jaeger - OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 - WEB_HTTP_PORT: 8181 - WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 8182 - WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_PROTOCOL_PORT: 8282 - WEB_HTTP_PROTOCOL_PATH: /protocol - WEB_HTTP_CONTROL_PORT: 8183 - WEB_HTTP_CONTROL_PATH: /control - EDC_CONTROL_ENDPOINT: http://provider:8183/control - EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol - EDC_PARTICIPANT_ID: provider - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /resources/README.md - entrypoint: java - -javaagent:/app/libs/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/resources/logging.properties - -jar /app/connector.jar - - jaeger: - image: jaegertracing/all-in-one - ports: - - "16686:16686" - - prometheus: - image: prom/prometheus:v2.30.3 - volumes: - - ./prometheus/:/etc/prometheus/ - ports: - - "9090:9090" diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json deleted file mode 100644 index b84b7631..00000000 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "dataDestination": { - "type": "File", - "path": "/resources/README_transferred.md", - "keyName": "keyName" - }, - "protocol": "dataspace-protocol-http", - "assetId": "test-document", - "contractId": "{agreement ID}", - "connectorId": "provider", - "connectorAddress": "http://provider:8282/protocol", - "privateProperties": {} -} diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile deleted file mode 100644 index 93680a50..00000000 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:jdk17 AS build - -WORKDIR /home/gradle/project/ -COPY --chown=gradle:gradle . /home/gradle/project/ -RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-consumer:build - -FROM openjdk:17-slim - -WORKDIR /app -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile deleted file mode 100644 index 50efc2f5..00000000 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM gradle:jdk17 AS build - -WORKDIR /home/gradle/project/ -COPY --chown=gradle:gradle . /home/gradle/project/ -RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-provider:build - -FROM openjdk:17-slim - -WORKDIR /app -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar -COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-06-consumer-pull-http/README.md b/transfer/transfer-06-consumer-pull-http/README.md deleted file mode 100644 index 23ebea8c..00000000 --- a/transfer/transfer-06-consumer-pull-http/README.md +++ /dev/null @@ -1,517 +0,0 @@ -# Implement a simple "Consumer Pull" Http transfer flow - -The purpose of this example is to show a data exchange between 2 connectors, one representing the -data provider and the other, the consumer. It's based on a consumer pull usecase that you can find -more details -on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane) -For the sake of simplicity, the provider and the consumer -will be on the same machine, but in a real world configuration, they will likely be on different -machines. The final goal of this example is to present the steps through which the 2 connectors will -have to pass so that the consumer can have access to the data, held by the provider. - -Those steps are the following: - -* Running the provider connector -* Running the consumer connector -* Running a Http server that will receive the Endpoint Data Reference on the consumer side, that - contains the url to be used to get the data. -* Register data plane instance for provider connector -* Register data plane instance for consumer connector -* Create an Asset on the provider (The asset will be the data to be shared) -* Create an access policy on the provider (The policy will define the access right to the data) -* Create a contract definition on the provider - -At this step, the consumer should be able to fetch the catalog from the provider and to see the -contract offer generated from the resources that have been created. - -Once the catalog is available, to access the data, the consumer should follow the following steps: - -* Performing a contract negotiation with the provider -* Performing a transfer - * The consumer will initiate a file transfer - * The provider will send an EndpointDataReference to the consumer -* The consumer could reach the endpoint and access the data - -Also, in order to keep things organized, the code in this example has been separated into several -Java modules: - -* `connector`: contains the configuration and build files for both the - consumer and the provider connector -* `backend-service`: represent the backend service where the consumer connector will send the - EndpointDataReference to access the data - -> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single -> asset. This will be deleted after the provider shutdown. - -### Provider connector - -The provider connector is the one providing EndpointDataReference to the consumer after it initiates -a transfer. - -### Consumer connector - -The consumer is the one "requesting" the data to the provider. - -# How to build a connector - -In fact, in the configuration of our example, both the provider and the consumer are connectors. -Therefore, to set up our example, we need to start a connector with the configuration for a provider -and another one with the configuration of a consumer. - -This section allows you to build the connector before launching it. - -```bash -./gradlew transfer:transfer-06-consumer-pull-http:http-pull-connector:build -``` - -After the build end you should verify that the connector jar is created in the directory -[http-pull-connector.jar](http-pull-connector/build/libs/http-pull-connector.jar) - -# How to run a connector - -It is important to note that only the property file differs between the consumer and the provider. -You can find the configuration file in the directories below: - -* [provider](http-pull-provider/provider-configuration.properties) -* [consumer](http-pull-consumer/consumer-configuration.properties) - -The section bellow will show you some explanation about some of the properties that you can find in -the configuration files. - -#### 1. edc.receiver.http.endpoint - -This property is used to define the endpoint where the connector consumer will send the -EndpointDataReference. - -#### 2. edc.dataplane.token.validation.endpoint - -This property is used to define the endpoint exposed by the control plane to validate the token. - -## Run the connectors - -To run the provider, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-06-consumer-pull-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-vault.properties -Dedc.fs.config=transfer/transfer-06-consumer-pull-http/http-pull-provider/provider-configuration.properties -jar transfer/transfer-06-consumer-pull-http/http-pull-connector/build/libs/pull-connector.jar -``` - -To run a consumer, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-06-consumer-pull-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-vault.properties -Dedc.fs.config=transfer/transfer-06-consumer-pull-http/http-pull-consumer/consumer-configuration.properties -jar transfer/transfer-06-consumer-pull-http/http-pull-connector/build/libs/pull-connector.jar -``` - -Assuming you didn't change the ports in config files, the consumer will listen on the -ports `29191`, `29192` (management API) and `29292` (DSP API) and the provider will listen on the -ports `12181`, `19182` (management API) and `19282` (DSP API). - -# Run the sample - -Running this sample consists of multiple steps, that are executed one by one and following the same -order. - -> Please in case you have some issues with the jq option, not that it's not mandatory, and you can -> drop it from the command. it's just used to format the output, and the same advice should be -> applied to all calls that use `jq`. - -### 1. Register data plane instance for provider - -Before a consumer can start talking to a provider, it is necessary to register the data plane -instance of a connector. This is done by sending a POST request to the management API of the -provider connector. The request body should contain the data plane instance of the consumer -connector. - -The registration of the provider data plane instance is done by sending a POST -request to the management API of the connector. - -```bash -curl -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ "HttpData" ], - "allowedDestTypes": [ "HttpProxy", "HttpData" ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } - }' \ - -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq -``` - -### 2. Create an asset on the provider side - -The provider connector needs to transfer a file to the location specified by the consumer connector -when the data are requested. In order to offer any data, the provider must maintain an internal list -of resources offered, through a contract offer, the so-called "catalog". - -The following request creates an asset on the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "asset": { - "@id": "assetId", - "properties": { - "name": "product description", - "contenttype": "application/json" - } - }, - "dataAddress": { - "type": "HttpData", - "name": "Test asset", - "baseUrl": "https://jsonplaceholder.typicode.com/users", - "proxyPath": "true" - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ - -s | jq -``` - -> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for -> the purpose of this example. It will be the data that the consumer will pull on the sample -> execution. - -Additional properties on `HttpData` can be used to allow consumers to enrich the data request: - -- `proxyPath`: allows specifying additional path segments. -- `proxyQueryParams`: allows specifying query params. -- `proxyBody`: allows attaching a body. -- `proxyMethod`: allows specifying the Http Method (default `GET`) - -### 3. Create a policy on the provider - -In order to manage the accessibility rules of an asset, it is essential to create a policy. However, -to keep things simple, we will choose a policy that gives direct access to all the assets that are -associated within the contract definitions. -This means that the consumer connector can request any asset from the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "aPolicy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ - -s | jq -``` - -### 4. Create a contract definition on provider - -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition -associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "aPolicy", - "contractPolicyId": "aPolicy", - "assetsSelector": [] - }' -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ - -s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "1", - "edc:createdAt": 1674578184023, - ... -} -``` - -### 5. How to fetch catalog on consumer side - -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain -all the contract offers available for negotiation. In our case, it will contain a single contract -offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following -endpoint: - -```bash -curl -X POST "http://localhost:29193/management/v2/catalog/request" \ - -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "providerUrl": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" - }' -s | jq -``` - -Sample output: - -```json -{ - "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", - "@type": "dcat:Catalog", - "dcat:dataset": { - "@id": "assetId", - "@type": "dcat:Dataset", - "odrl:hasPolicy": { - "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@type": "odrl:Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - }, - "dcat:distribution": [ - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpProxy" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - }, - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpData" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - } - ], - "edc:name": "product description", - "edc:id": "assetId", - "edc:contenttype": "application/json" - }, - "dcat:service": { - "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", - "@type": "dcat:DataService", - "dct:terms": "connector", - "dct:endpointUrl": "http://localhost:19194/protocol" - }, - "edc:participantId": "anonymous", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 6. Negotiate a contract - -In order to request any data, a contract gets negotiated, and an agreement is resulting has to be -negotiated between providers and consumers. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence -looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the - provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later - reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counter offers in addition to just confirming or declining an offer. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "offer": { - "offerId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "assetId": "assetId", - "policy": { - "@id": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@type": "Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - } - } -}' -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ --s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", - "edc:createdAt": 1685525281848, - ... -} -``` - -### 7. Getting the contract agreement id - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. -This UUID is the ID of the ongoing contract negotiation between consumer and provider. The -negotiation sequence between provider and consumer is executed asynchronously in the background by a -state machine. Once both provider and consumer either reach the `confirmed` or the `declined` -state, the negotiation is finished. We can now use the UUID to check the current status of the -negotiation using an endpoint on the consumer side. - -```bash -curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ - --header 'Content-Type: application/json' \ - -s | jq -``` - -Sample output: - -```json -{ - "@type": "edc:ContractNegotiationDto", - "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", - "edc:type": "CONSUMER", - "edc:protocol": "dataspace-protocol-http", - "edc:state": "FINALIZED", - "edc:counterPartyAddress": "http://localhost:19194/protocol", - "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 8. Start the transfer - -As a pre-requisite, you need to have an http server that runs on port 4000 and logs all the incoming requests, it will -be mandatory to get the EndpointDataReference that will be used to get the data. - -```bash -./gradlew util:http-request-logger:build -HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar -``` - -Now that we have a contract agreement, we can finally request the file. In the request body, we need -to specify which asset we want transferred, the ID of the contract agreement, the address of the -provider connector and where we want the file transferred. You will find the request body below. -Before executing the request, insert the contract agreement ID from the previous step. Then run : - -> the "HttpProxy" method is used for the consumer pull method, and it means that it will be up to -> the consumer to request the data to the provider and that the request will be a proxy for the -> datasource - -```bash -curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ - -H "Content-Type: application/json" \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "contractId": "", - "assetId": "assetId", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpProxy" - } - }' \ - -s | jq -``` - -Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( -process id) created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and -performed asynchronously. - -Sample output: - -```json -{ - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:createdAt": 1674078357807, - ... -} -``` - -### 9. Check the transfer status - -Due to the nature of the transfer, it will be very fast and most likely already done by the time you -read the UUID. - -```bash -curl http://localhost:29193/management/v2/transferprocesses/ -``` - - -You should see the Transfer Process in `COMPLETED` state: -```json -{ - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:state": "COMPLETED", - ... -} - -``` - -### 10. Pull the data - -At this step, if you look at the http server logs, you will find a json representing the EndpointDataReference, needed -to get the data from the provider: - -```json -{ - "id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "endpoint": "http://localhost:29291/public/", - "authKey": "Authorization", - "authCode": "eyJhbGciOiJSUzI1NiJ9.eyJkYWQiOiJ7XCJwcm9wZXJ0aWVzXCI6e1wiYXV0aEtleVwiOlwiQXV0aG9yaXphdGlvblwiLFwiYmFzZVVybFwiOlwiaHR0cDpcL1wvbG9jYWxob3N0OjE5MjkxXC9wdWJsaWNcL1wiLFwiYXV0aENvZGVcIjpcImV5SmhiR2NpT2lKU1V6STFOaUo5LmV5SmtZV1FpT2lKN1hDSndjbTl3WlhKMGFXVnpYQ0k2ZTF3aVltRnpaVlZ5YkZ3aU9sd2lhSFIwY0hNNlhDOWNMMnB6YjI1d2JHRmpaV2h2YkdSbGNpNTBlWEJwWTI5a1pTNWpiMjFjTDNWelpYSnpYQ0lzWENKdVlXMWxYQ0k2WENKVVpYTjBJR0Z6YzJWMFhDSXNYQ0owZVhCbFhDSTZYQ0pJZEhSd1JHRjBZVndpZlgwaUxDSmxlSEFpT2pFMk56UTFPRGcwTWprc0ltTnBaQ0k2SWpFNk1XVTBOemc1TldZdE9UQXlOUzAwT1dVeExUazNNV1F0WldJNE5qVmpNemhrTlRRd0luMC5ITFJ6SFBkT2IxTVdWeWdYZi15a0NEMHZkU3NwUXlMclFOelFZckw5eU1tQjBzQThwMHFGYWV0ZjBYZHNHMG1HOFFNNUl5NlFtNVU3QnJFOUwxSE5UMktoaHFJZ1U2d3JuMVhGVUhtOERyb2dSemxuUkRlTU9ZMXowcDB6T2MwNGNDeFJWOEZoemo4UnVRVXVFODYwUzhqbU4wZk5sZHZWNlFpUVFYdy00QmRTQjNGYWJ1TmFUcFh6bDU1QV9SR2hNUGphS2w3RGsycXpJZ0ozMkhIdGIyQzhhZGJCY1pmRk12aEM2anZ2U1FieTRlZXU0OU1hclEydElJVmFRS1B4ajhYVnI3ZFFkYV95MUE4anNpekNjeWxyU3ljRklYRUV3eHh6Rm5XWmczV2htSUxPUFJmTzhna2RtemlnaXRlRjVEcmhnNjZJZzJPR0Eza2dBTUxtc3dcIixcInByb3h5TWV0aG9kXCI6XCJ0cnVlXCIsXCJwcm94eVF1ZXJ5UGFyYW1zXCI6XCJ0cnVlXCIsXCJwcm94eUJvZHlcIjpcInRydWVcIixcInR5cGVcIjpcIkh0dHBEYXRhXCIsXCJwcm94eVBhdGhcIjpcInRydWVcIn19IiwiZXhwIjoxNjc0NTg4NDI5LCJjaWQiOiIxOjFlNDc4OTVmLTkwMjUtNDllMS05NzFkLWViODY1YzM4ZDU0MCJ9.WhbTzERmM75mNMUG2Sh-8ZW6uDQCus_5uJPvGjAX16Ucc-2rDcOhAxrHjR_AAV4zWjKBHxQhYk2o9jD-9OiYb8Urv8vN4WtYFhxJ09A0V2c6lB1ouuPyCA_qKqJEWryTbturht4vf7W72P37ERo_HwlObOuJMq9CS4swA0GBqWupZHAnF-uPIQckaS9vLybJ-gqEhGxSnY4QAZ9-iwSUhkrH8zY2GCDkzAWIPmvtvRhAs9NqVkoUswG-ez1SUw5bKF0hn2OXv_KhfR8VsKKYUbKDQf5Wagk7rumlYbXMPNAEEagI4R0xiwKWVTfwwZPy_pYnHE7b4GQECz3NjhgdIw", - "properties": { - "cid": "1:1e47895f-9025-49e1-971d-eb865c38d540" - } -} -``` - -Once this json is read, use a tool like postman or curl to execute the following query, to read the -data - -```bash -curl --location --request GET 'http://localhost:29291/public/' --header 'Authorization: ' -``` - -At the end, and to be sure that you correctly achieved the pull, you can check if the data you get -is the same as the one you can get at https://jsonplaceholder.typicode.com/users - - -Since we configured the `HttpData` with `proxyPath`, we could also ask for a specific user with: - -```bash -curl --location --request GET 'http://localhost:29291/public/1' --header 'Authorization: ' -``` - -And the data returned will be the same as in https://jsonplaceholder.typicode.com/users/1 diff --git a/transfer/transfer-07-provider-push-http/README.md b/transfer/transfer-07-provider-push-http/README.md deleted file mode 100644 index a6769761..00000000 --- a/transfer/transfer-07-provider-push-http/README.md +++ /dev/null @@ -1,445 +0,0 @@ -# Implement a simple "Provider Push" Http transfer flow - -The purpose of this example is to show a data exchange between two connectors, one representing the -data provider and the other, the consumer. It's based on a provider push use case that you can find -more details -on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). -For the sake of simplicity, the provider and the consumer -will be on the same machine, but in a real world configuration, they will likely be on different -machines. The final goal of this example is to present the steps through which the two connectors will -have to pass so that the consumer can have access to the data, held by the provider. - -Those steps are the following: - -* Running the provider connector -* Running the consumer connector -* Register data plane instance for provider connector -* Create an Asset on the provider (The asset will be the data to be shared) -* Create an access policy on the provider (The policy will define the access right to the data) -* Create a contract definition on the provider - -At this step, the consumer should be able to fetch the catalog from the provider and to see the -contract offer generated from the resources that have been created. - -Once the catalog is available, to access the data, the sample will go through: - -* Performing a contract negotiation with the provider -* Performing a transfer - * The consumer will initiate a file transfer - * Provider Control Plane retrieves the DataAddress of the actual data source and creates a - DataFlowRequest based on the received DataRequest and this data address -* Provider Data Plane fetches data from the actual data source -* Provider Data Plane pushes data to the consumer service - -Also, in order to keep things organized, the code in this example has been separated into one -Java module: - -* `connector`: contains the configuration and build files for both the - consumer and the provider connector -* `provider-push-backend-service`: represent the backend service where the provider will push the data after the - consumer has initiated the transfer - -> For the sake of simplicity, we will use an in-memory catalog and fill it with just one single -> asset. This will be deleted after the provider shutdown. - -# How to build a connector - -In fact, in the configuration of our example, both the provider and the consumer are connectors. -Therefore, to set up our example, we need to start a connector with the configuration for a provider -and another one with the configuration of a consumer. - -This section allows you to build the connector before launching it. - -```bash -./gradlew transfer:transfer-07-provider-push-http:http-push-connector:build -``` - -After the building end, you should verify that the connector jar is created in the directory -[http-push-connector.jar](http-push-connector/build/libs/http-push-connector.jar) - -# How to run a connector - -It is important to note that only the property file differs between the consumer and the provider. -You can find the configuration file in the directories below: - -* [provider](http-push-provider/provider-configuration.properties) -* [consumer](http-push-consumer/consumer-configuration.properties) - -The section bellow will show you some explanation about some of the properties that you can find in -the configuration files. - -## Run the connectors - -To run the provider, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-07-provider-push-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-07-provider-push-http/http-push-provider/provider-vault.properties -Dedc.fs.config=transfer/transfer-07-provider-push-http/http-push-provider/provider-configuration.properties -jar transfer/transfer-07-provider-push-http/http-push-connector/build/libs/push-connector.jar -``` - -To run the consumer, just run the following command - -```bash -java -Dedc.keystore=transfer/transfer-07-provider-push-http/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.vault=transfer/transfer-07-provider-push-http/http-push-consumer/consumer-vault.properties -Dedc.fs.config=transfer/transfer-07-provider-push-http/http-push-consumer/consumer-configuration.properties -jar transfer/transfer-07-provider-push-http/http-push-connector/build/libs/push-connector.jar -``` - -Assuming you didn't change the ports in config files, the consumer will listen on the -ports `29191`, `29193` (management API) and `29194` (DSP API) and the provider will listen on the -ports `19191`, `19193` (management API) and `19194` (DSP API). - -# Run the sample - -Running this sample consists of multiple steps, that are executed one by one and following the same -order. - -> Please in case you have some issues with the jq option, not that it's not mandatory, and you can -> drop it from the command. it's just used to format the output, and the same advice should be -> applied to all calls that use `jq`. - -### 1. Register data plane instance for provider - -Before a consumer can start talking to a provider, it is necessary to register the data plane -instance of a connector. This is done by sending a POST request to the management API of the -provider connector. The request body should contain the data plane instance of the consumer -connector. - -The registration of the provider data plane instance is done by sending a POST -request to the management API of the connector. - -```bash -curl -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ "HttpData" ], - "allowedDestTypes": [ "HttpProxy", "HttpData" ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } - }' \ - -X POST "http://localhost:19193/management/v2/dataplanes" -``` - -### 2. Create an Asset on the provider side - -The provider connector needs to transfer a file to the location specified by the consumer connector -when the data are requested. In order to offer any data, the provider must maintain an internal list -of resources offered, through a contract offer, the so-called "catalog". - -The following request creates an asset on the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "asset": { - "@id": "assetId", - "properties": { - "name": "product description", - "contenttype": "application/json" - } - }, - "dataAddress": { - "name": "Test asset", - "baseUrl": "https://jsonplaceholder.typicode.com/users", - "type": "HttpData" - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/assets \ - -s | jq -``` - -> It is important to note that the `baseUrl` property of the `dataAddress` is a fake data used for -> the purpose of this example. It will be the data that the provider will push on the sample -> execution. - -### 3. Create a Policy on the provider - -In order to manage the accessibility rules of an asset, it is essential to create a policy. However, -to keep things simple, we will choose a policy that gives direct access to all the assets that are -associated within the contract definitions. -This means that the consumer connector can request any asset from the provider connector. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "aPolicy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } - }' -H 'content-type: application/json' http://localhost:19193/management/v2/policydefinitions \ - -s | jq -``` - -### 4. Create a contract definition on Provider - -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition -associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "aPolicy", - "contractPolicyId": "aPolicy", - "assetsSelector": [] - }' -H 'content-type: application/json' http://localhost:19193/management/v2/contractdefinitions \ - -s | jq -``` - -Sample output: - -```json -{ - "createdAt": 1674578184023, - "id": "1" -} -``` - -### 5. How to fetch catalog on consumer side - -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain -all the contract offers available for negotiation. In our case, it will contain a single contract -offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following -endpoint: - -```bash -curl -X POST "http://localhost:29193/management/v2/catalog/request" \ - -H 'Content-Type: application/json' \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "providerUrl": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" - }' -s | jq -``` - -Sample output: - -```json -{ - "@id": "31f6d748-d35b-4dec-9e34-d141fd17b458", - "@type": "dcat:Catalog", - "dcat:dataset": { - "@id": "assetId", - "@type": "dcat:Dataset", - "odrl:hasPolicy": { - "@id": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "@type": "odrl:Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - }, - "dcat:distribution": [ - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpProxy" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - }, - { - "@type": "dcat:Distribution", - "dct:format": { - "@id": "HttpData" - }, - "dcat:accessService": "2a5178c3-c937-4ac2-85be-c46dbc6c5642" - } - ], - "edc:name": "product description", - "edc:id": "assetId", - "edc:contenttype": "application/json" - }, - "dcat:service": { - "@id": "2a5178c3-c937-4ac2-85be-c46dbc6c5642", - "@type": "dcat:DataService", - "dct:terms": "connector", - "dct:endpointUrl": "http://localhost:19194/protocol" - }, - "edc:participantId": "anonymous", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 6. Negotiate a contract - -In order to request any data, a contract gets negotiated, and an agreement is resulting has to be -negotiated between providers and consumers. - -The consumer now needs to initiate a contract negotiation sequence with the provider. That sequence -looks as follows: - -1. Consumer sends a contract offer to the provider (__currently, this has to be equal to the - provider's offer!__) -2. Provider validates the received offer against its own offer -3. Provider either sends an agreement or a rejection, depending on the validation result -4. In case of successful validation, provider and consumer store the received agreement for later - reference - -Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counteroffers in addition to just confirming or declining an offer. - -```bash -curl -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "offer": { - "offerId": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "assetId": "assetId", - "policy": { - "@id": "MQ==:YXNzZXRJZA==:MzY3ZGZlYTgtYWI5OS00OWMwLThmNmYtM2Y2YmMxNGE1ZDc4", - "@type": "Set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [], - "odrl:target": "assetId" - } - } -}' -X POST -H 'content-type: application/json' http://localhost:29193/management/v2/contractnegotiations \ --s | jq -``` - -Sample output: - -```json -{ - ... - "@id": "254015f3-5f1e-4a59-9ad9-bf0e42d4819e", - "edc:createdAt": 1685525281848, - ... -} -``` - -### 7. Getting the contract agreement id - -After calling the endpoint for initiating a contract negotiation, we get a UUID as the response. -This UUID is the ID of the ongoing contract negotiation between consumer and provider. The -negotiation sequence between provider and consumer is executed asynchronously in the background by a -state machine. Once both provider and consumer either reach the `confirmed` or the `declined` -state, the negotiation is finished. We can now use the UUID to check the current status of the -negotiation using an endpoint on the consumer side. - -```bash -curl -X GET "http://localhost:29193/management/v2/contractnegotiations/" \ - --header 'Content-Type: application/json' \ - -s | jq -``` - -Sample output: - -```json -{ - "@type": "edc:ContractNegotiationDto", - "@id": "5ca21b82-075b-4682-add8-c26c9a2ced67", - "edc:type": "CONSUMER", - "edc:protocol": "dataspace-protocol-http", - "edc:state": "FINALIZED", - "edc:counterPartyAddress": "http://localhost:19194/protocol", - "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:MjQ2ODMxOTMtZmFhMS00MzMxLWE2YzYtYTQ1ZjNkNzJkYWNk", - "@context": { - "dct": "https://purl.org/dc/terms/", - "edc": "https://w3id.org/edc/v0.0.1/ns/", - "dcat": "https://www.w3.org/ns/dcat/", - "odrl": "http://www.w3.org/ns/odrl/2/", - "dspace": "https://w3id.org/dspace/v0.8/" - } -} -``` - -### 8. Start the transfer - -As a pre-requisite, you need to have an http server that runs on port 4000 and logs all the incoming requests, it will -be mandatory to get the data from the provider. - -```bash -./gradlew util:http-request-logger:build -HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar -``` - -Now that we have a contract agreement, we can finally request the file. In the request body, we need -to specify which asset we want transferred, the ID of the contract agreement, the address of the -provider connector and where we want the file transferred. You will find the request body below. -Before executing the request, insert the contract agreement ID from the previous step. Then run : - -> keep in mind that, to make a transfer with a provider push method, the dataDestination type should -> be any value different from the "HttpProxy". - -```bash -curl -X POST "http://localhost:29193/management/v2/transferprocesses" \ - -H "Content-Type: application/json" \ - -d '{ - "@context": { - "edc": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "connectorAddress": "http://localhost:19194/protocol", - "contractId": "", - "assetId": "assetId", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpData", - "baseUrl": "http://localhost:4000/api/consumer/store" - } - }' \ - -s | jq -``` - -Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` ( -process id) created on the consumer -side, because like the contract negotiation, the data transfer is handled in a state machine and -performed asynchronously. - -Sample output: - -```json - { - ... - "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", - "edc:createdAt": 1674078357807, - ... -} -``` - -### 9. Check the transfer status - -Due to the nature of the transfer, it will be very fast and most likely already done by the time you -read the UUID. - -```bash -curl http://localhost:29193/management/v2/transferprocesses/ -``` - -### 10. Check the data - -At this step, you can check the data by checking the log of the http server exposed on port 4000, you should see a log -that shows the same data that you can get from https://jsonplaceholder.typicode.com/users diff --git a/transfer/transfer-07-provider-push-http/certs/cert.pem b/transfer/transfer-07-provider-push-http/certs/cert.pem deleted file mode 100644 index c7dc26fa..00000000 --- a/transfer/transfer-07-provider-push-http/certs/cert.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy -MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g -E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS -PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H -I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W -EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 -h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud -DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B -2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn -QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ -rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe -Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy -+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR -IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ -g3h+15GuzbsSzOCOEYOT ------END CERTIFICATE----- diff --git a/transfer/transfer-07-provider-push-http/certs/cert.pfx b/transfer/transfer-07-provider-push-http/certs/cert.pfx deleted file mode 100644 index 7ac9c73e0eb6b5c2fa625d818e42d55bdafc8a2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2477 zcmY+^c{J3G8V7K*W@N}R7=$7FHnNl@TSGL2Fj;DNOC*GhE&IP}zqvy)xzv zQL;p;A(BR~v1ZAx(5-Xsz3;t$Jm)#*`#tCR@A+T>L=6}O!2$?>I6^7OB54~5Is~Es z2zMxexcDn}!U9m>;!u#NoRD=ju#8QFzm^2$y49g1TMv z=T@sTo@Iuph_A%yU=lxc7M{sJMHUT>F|}KXUgo$hFFS>A2!j7*6*i&bNm83Pw2;<1 zlD}V5KUbh&o|jOxsP+BiwXBG4&`tucGj8`-Hm@xYJ}`H^&q$4GmUu<1YCBvj`C$0i zkFsaL3jy!*E?2AUSW6Pe-Zjg?APSh+Uhq-Q)U@7jmt#8E>T$EH6!|pKz~;1w=$~nhB-?cONKvYWB(}r)QbC0-GdWIqew0w+G5Wq#m^rPtZ{$ z&t3^ej`3$~G<<2pp49+5+q{j;Rh)KZXRnVO3qF5>eHL=xcsdXrkEpgxwm0V8`ssk6Z8vc4Et<{a&YT958$yF6j}K( z4hN^-U5JL_{5>%{Y?3HMe0}$$xMM|0Q~;4aZqhSedSM+=_Eg(Xx&_xr&h4{kQ6=Zy zW@OQ7;(7v+Fweo zb}gS9>{w6uxhvyvlziQ9z-3d+n125oDs)1&%G zEwA3(y*}qIWtKDR8WA@gI9QkkDNyw1yM+2Q51Ffz7>M>4w=m^={|4pZ%S)yOfKw=S zz69`I8q8wN^N`NiDcpnDll_@y_Lt-uFCKP{LaIrnK#~AY;j~X%jH{UC%6?YFF(6`J zeO0vTRy6aht=iu+Gn1WZZ9${yya0`i>EAky_j=kWwxgJ@qBH-e;dkS@+_oyS3!Aiw z;flVSZ2Gwg5uLIPua4c7LXmk++;IQ)ShM>oI!v-Ny??463&2eN=kX~3>^KyFsr`zT zevKUb@P8QK0)c;dvHXjWh5yq5@>_>7Tkb5#{9e;<9iRa8RoC_4ISv`^=M(qQdQ#2V zIza+e3IWHj4>i1V?+l92n8{c(>>O7{qXrb~$} z8V6i+B42m1p07qSh%@E`sm?9n2@PvNYy1jX*e|%F=6kUedQy;Iln(yLlX1`aN#?b- zZwXNhKWgoMN~ESsM{=(WNU3B-P_1gdul?TFtyYb>%F_uP1wj4yp>Z!i6*k*Ul%Gz)g&^5L%mg;3Yei{>pW3}~9}Fie!CscOnc zJgrKau67c@sA=-R{3>m^F}gVEM{D)BD;Rfv&5G3Fy=WURVqP6QJxj82TTmc;y4Q1- zx|64G<-u>)G8wNcEv5P}fF>N`2APM^GsZsD9(GGFY8a1t-ibjD#{`UD_>(v-XlYzo zA;v?EMrJqVD#h=r6cyea-Ed8TH>Zq;HKlDnF#HM0hMbMSu#w}iv=RZVfG{~?FaJbD zMa^hLzlX||5&8d(&Eyl0P5RU*~LQSx?Wln6}c1 zjE%SIZYd-En@6!L4H>W9OOz7Uh%swV~-_xx9L37=5~uCx-Us8{Wfv z!)kuI6u*;E^(MO0p!ubmp{22ExfFD%2=4oC9Gi#0bMN>{7MOeNuCxxli^wSO z#`0sH5ncdKZJqBDq$!_D$m6M1M9n|sZbu%mtl$5t$fH@XWMBH^*&PYhQKlKIzN#AS zP+p19Vo>>tV1Zg+=2Mjn;E1Epntc)Ceohe56KASXPy5a+R<=Vut-5#zROAXC2-6Qq!-V_HxY5dt)>gm+(P2v_EXSK~h`}Bm!(8j7N zDp}fG?p^A7UZ|tg4P;$YSJ-!8MYi_%t?<-l^SUW^yH23Ug-IiYTF2H#qp7+WS4$R* zaAti?s5msEyR0#3X@J~V5>*x`)cC0CVNqh9tGDxxLcyCjv}KFV2j%^Du{Tt5oE7^+ z;-xx^OdO|Xz6kXWC+|Nr=h33y@q4B#HOS`(w#wQyP1IL;yd7dDd?TSkv`?1vgrI`tLtzxEjlPjW_Z(lo%ao_L5vmNUfyH1EaF{X=1S|^U nfC>)L0b!m6KEg{ { manifest { attributes["Main-Class"] = "org.eclipse.edc.samples.util.HttpRequestLoggerServer" } + archiveFileName.set("http-request-logger.jar") } From 927a56f6feb1a92556d7e27e6461517a92b404a4 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Mon, 13 Nov 2023 16:55:31 +0900 Subject: [PATCH 20/35] docs: fix invalid command line in the README of transfer-00-prerequisites (#149) Signed-off-by: Masatake Iwasaki --- transfer/transfer-00-prerequisites/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md index 4e8a968d..d663a21b 100644 --- a/transfer/transfer-00-prerequisites/README.md +++ b/transfer/transfer-00-prerequisites/README.md @@ -88,9 +88,9 @@ Open a new terminal and execute: ```bash curl -H 'Content-Type: application/json' \ -d @transfer/transfer-00-prerequisites/resources/dataplane/register-data-plane-provider.json \ - -X POST "http://localhost:19193/management/v2/dataplanes" | -s | jq + -X POST "http://localhost:19193/management/v2/dataplanes" -s | jq ``` The connectors have been configured successfully and are ready to be used. -[Next Chapter](../transfer-01-negotiation/README.md) \ No newline at end of file +[Next Chapter](../transfer-01-negotiation/README.md) From 8b820640cde63008d532204b7aa3edf27e99afd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:55:55 +0100 Subject: [PATCH 21/35] build(deps): bump agilepathway/label-checker from 1.6.7 to 1.6.9 (#147) Bumps [agilepathway/label-checker](https://github.com/agilepathway/label-checker) from 1.6.7 to 1.6.9. - [Release notes](https://github.com/agilepathway/label-checker/releases) - [Commits](https://github.com/agilepathway/label-checker/compare/v1.6.7...v1.6.9) --- updated-dependencies: - dependency-name: agilepathway/label-checker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scan-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scan-pull-request.yml b/.github/workflows/scan-pull-request.yml index 4175ac06..7ddea226 100644 --- a/.github/workflows/scan-pull-request.yml +++ b/.github/workflows/scan-pull-request.yml @@ -28,7 +28,7 @@ jobs: continue-on-error: false steps: - uses: actions/checkout@v4 - - uses: agilepathway/label-checker@v1.6.7 + - uses: agilepathway/label-checker@v1.6.9 with: any_of: api,bug,build,dependencies,documentation,enhancement,refactoring repo_token: ${{ secrets.GITHUB_TOKEN }} From cf5678784a2c5dca6116175243007ff086416dac Mon Sep 17 00:00:00 2001 From: Ege Korkan Date: Mon, 13 Nov 2023 09:39:08 +0100 Subject: [PATCH 22/35] fix(docs): change contributing.md link to correct one (#145) * fix(docs): change contributing.md link to correct one * fix(docs): point to common repo readme rather than same repo * fix(docs): delete contributing.md since we point to core repo now --- CONTRIBUTING.md | 4 ---- README.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 9d78b778..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,4 +0,0 @@ -# Contributing to the Project - -Thank you for your interest in contributing to this project. Please refer to the -[contributing guidelines of the Eclipse Dataspace Connector project](https://github.com/eclipse-edc/Connector/blob/main/CONTRIBUTING.md). diff --git a/README.md b/README.md index 865b99fb..f33195c5 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ All transfer samples are located in the `advanced` directory. ## Contributing -See [how to contribute](CONTRIBUTING.md). +See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). ## License From e455513cad5141adf650ea817b74ec5b1e87f98b Mon Sep 17 00:00:00 2001 From: farhin23 Date: Tue, 21 Mar 2023 10:07:52 +0100 Subject: [PATCH 23/35] feat: create basic policy sample --- .../policy-01-contract-negotiation/README.md | 147 ++++++++++++++++++ .../contractoffer.json | 70 +++++++++ .../build.gradle.kts | 34 ++++ .../config.properties | 14 ++ .../build.gradle.kts | 14 ++ .../policy/PolicyFunctionsExtension.java | 143 +++++++++++++++++ .../policy/TimeIntervalFunction.java | 48 ++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + 8 files changed, 471 insertions(+) create mode 100644 policy/policy-01-contract-negotiation/README.md create mode 100644 policy/policy-01-contract-negotiation/contractoffer.json create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java create mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/policy/policy-01-contract-negotiation/README.md b/policy/policy-01-contract-negotiation/README.md new file mode 100644 index 00000000..5778a1da --- /dev/null +++ b/policy/policy-01-contract-negotiation/README.md @@ -0,0 +1,147 @@ +# Create a policy function + +This sample shows how a custom function for policy evaluation can be implemented and registered. For simplicity, +we will use just one connector, which initiates a contract negotiation with itself. The sample contains two modules: + +- `policy-contract-negotiation-connector`: contains the connector configuration +- `policy-contract-negotiation-policy-functions`: provides the function for policy evaluation + +## Creating and registering the function + +The [`TimeIntervalFunction`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java) +implements the `AtomicConstraintFunction` interface, which contains a single method for evaluating a constraint. +In that method, the `operator` and `right value` of the constraint can be used for evaluation. In this example, we +want to check whether the evaluation happens within a specified time interval. So depending on whether the operator is +`GT` or `LT`, we check if the current date is before or after the date specified in the `right value`. + +```java +@Override +public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + Date date; + try { + date = sdf.parse((String) rightValue); + } catch (ParseException e) { + monitor.severe("Failed to parse right value of constraint to date."); + return false; + } + + switch (operator) { + case LT: var isBefore = new Date().before(date); + monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date."); + return isBefore; + case GT: var isAfter = new Date().after(date); + monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date."); + return isAfter; + default: return false; + } +} +``` + +Next, we have to register our function with the `PolicyEngine` and bind the desired action as well as the key used to +register our function to the desired scopes using the `RuleBindingRegistry`. This is done in the +[`PolicyFunctionsExtension`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java): + +```java +private final String policyTimeKey = "POLICY_EVALUATION_TIME"; + +//... + +@Override +public void initialize(ServiceExtensionContext context) { + //... + + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); + policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); + + //... +} +``` + +## Defining the policy + +Next to registering the function and creating the binding, the `PolicyFunctionsExtension` also creates an asset, +a policy and a contract definition linking the former two. The asset is the same one that is used in +sample [`transfer-01-file-transfer`](transfer/transfer-01-file-transfer/README.md), but this time a different policy is created. + +We start by creating the constraints, where we define that the policy evaluation time must be within a certain +time interval. We read the start and end dates for the interval from the settings. Then, we create two constraints: +one specifying that the evaluation time should be after our defined start date, and one specifying that the evaluation +time should be before our defined end date. **We need to set the constraints' left operands to the same key +that we previously used to register our function. Otherwise, the function will not be used to evaluate these +constraints.** + +```java +var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); +var notBeforeConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.GT) + .rightExpression(new LiteralExpression(startDate)) + .build(); + +var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T23:59:00.000+02:00"); +var notAfterConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.LT) + .rightExpression(new LiteralExpression(endDate)) + .build(); +``` + +Then, we create a `Permission` with action type `USE` and the two constraints. We use this permission to create and +store a policy. And last, we create the `ContractDefinition`. For the access policy, we use the same use-policy that is +used in sample `transfer:transfer-01-file-transfer`, and for the contract policy, we use the previously created policy with the time +interval restriction. We set the `AssetSelectorExpression` so that the contract definition is valid for our asset. + +## How to run the sample + +To see the policy enforcement working, this sample should be run twice with different configurations. + +### Configuration + +Choose one of the following configurations. Depending on which configuration you choose, the contract negotiation +will either be confirmed or declined. + +#### 1. Policy fulfilled + +Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is within +the defined interval. + +```properties +edc.samples.uc.constraint.date.start=2023-01-01T00:00:00.000+02:00 +edc.samples.uc.constraint.date.end=2023-12-31T23:59:00.000+02:00 +``` + +#### 2. Policy not fulfilled + +Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is +**not** within the defined interval. + +```properties +edc.samples.policy-01.constraint.date.start=2022-01-01T00:00:00.000+02:00 +edc.samples.policy-01.constraint.date.end=2022-12-31T23:59:00.000+02:00 +``` + +### Run the sample + +First, build and run the connector for this sample: + +```shell +./gradlew policy:policy-01-contract-negotiation:policy-contract-negotiation-connector:build +java -Dedc.fs.config=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar +#for windows: +java -D"edc.fs.config"=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar +``` + +Next, initiate a contract negotiation. The request body is prepared in [`contractoffer.json`](policy/policy-01-contract-negotiation/contractoffer.json). +In that file, replace the dates for the constraints with the dates you used in the `config.properties` file and then +run: + +```shell +curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @samples/uc-workshop/contractoffer.json "http://localhost:8182/api/v1/management/contractnegotiations" +``` + +Depending on the configuration you chose, you can see the contract negotiation either being confirmed or declined +in the connector logs. The logs also contain messages logged by the `TimeIntervalFunction`, which tell whether the +current date is before or after the start and end dates defined in the policy. diff --git a/policy/policy-01-contract-negotiation/contractoffer.json b/policy/policy-01-contract-negotiation/contractoffer.json new file mode 100644 index 00000000..9af961ad --- /dev/null +++ b/policy/policy-01-contract-negotiation/contractoffer.json @@ -0,0 +1,70 @@ +{ + "connectorId": "provider", + "connectorAddress": "http://localhost:8282/api/v1/ids/data", + "protocol": "ids-multipart", + "offer": { + "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "assetId": "test-document", + "policy": { + "uid": "956e172f-2de1-4501-8881-057a57fd0e69", + "permissions": [ + { + "edctype": "dataspaceconnector:permission", + "uid": null, + "target": "test-document", + "action": { + "type": "USE", + "includedIn": null, + "constraint": null + }, + "assignee": null, + "assigner": null, + "constraints": [ + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2023-01-01T00:00:00.000+02:00" + }, + "operator": "GT" + }, + { + "edctype": "AtomicConstraint", + "leftExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "POLICY_EVALUATION_TIME" + }, + "rightExpression": { + "edctype": "dataspaceconnector:literalexpression", + "value": "2023-12-31T00:00:00.000+02:00" + }, + "operator": "LT" + } + ], + "duties": [] + } + ], + "prohibitions": [], + "obligations": [], + "extensibleProperties": {}, + "inheritsFrom": null, + "assigner": null, + "assignee": null, + "target": null, + "@type": { + "@policytype": "set" + } + }, + "asset": { + "properties": { + "ids:byteSize": null, + "asset:prop:id": "test-document", + "ids:fileName": null + } + } + } +} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts new file mode 100644 index 00000000..a60d9218 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts @@ -0,0 +1,34 @@ +plugins { + `java-library` + id("application") + id("com.github.johnrengelman.shadow") version "7.1.2" +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + implementation("$groupId:control-plane-core:$edcVersion") + + implementation("$groupId:api-observability:$edcVersion") + + implementation("$groupId:configuration-filesystem:$edcVersion") + implementation("$groupId:iam-mock:$edcVersion") + + implementation("$groupId:auth-tokenbased:$edcVersion") + implementation("$groupId:management-api:$edcVersion") + + implementation("$groupId:ids:$edcVersion") + + implementation(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + exclude("**/pom.properties", "**/pom.xm") + mergeServiceFiles() + archiveFileName.set("connector.jar") +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties new file mode 100644 index 00000000..4a9f967f --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties @@ -0,0 +1,14 @@ +web.http.port=8181 +web.http.path=/api + +web.http.management.port=8182 +web.http.management.path=/api/v1/management + +edc.api.auth.key=password +ids.webhook.address=http://localhost:8282 +edc.ids.id=urn:connector:provider + +edc.samples.policy-01.asset.path = /path/to/file + +edc.samples.policy-01.constraint.date.start=2023-01-01T00:00:00.000+02:00 +edc.samples.policy-01.constraint.date.end=2023-12-31T00:00:00.000+02:00 diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts new file mode 100644 index 00000000..df01ec91 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` + id("application") +} + +val groupId: String by project +val edcVersion: String by project + +dependencies { + api("$groupId:data-plane-spi:$edcVersion") + + implementation("$groupId:control-plane-core:$edcVersion") + +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java new file mode 100644 index 00000000..a2dca9f6 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -0,0 +1,143 @@ +package org.eclipse.edc.sample.extension.policy; + +import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; +import org.eclipse.edc.connector.contract.spi.types.offer.ContractDefinition; +import org.eclipse.edc.connector.policy.spi.PolicyDefinition; +import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; +import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; +import org.eclipse.edc.policy.model.*; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.asset.AssetIndex; +import org.eclipse.edc.spi.asset.AssetSelectorExpression; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.asset.Asset; + +import java.nio.file.Path; + +import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; + +public class PolicyFunctionsExtension implements ServiceExtension { + private final String policyTimeKey = "POLICY_EVALUATION_TIME"; + private final String policyStartDateSetting = "edc.samples.policy-01.constraint.date.start"; + private final String policyEndDateSetting = "edc.samples.policy-01.constraint.date.end"; + private final String EDC_ASSET_PATH = "edc.samples.policy-01.asset.path"; + + @Inject + private RuleBindingRegistry ruleBindingRegistry; + + @Inject + private PolicyEngine policyEngine; + + @Inject + private PolicyDefinitionStore policyStore; + + @Inject + private ContractDefinitionStore contractDefinitionStore; + + @Inject + private AssetIndex assetIndex; + + @Override + public String name() { + return "Policy - contract-negotiation policies"; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); + policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); + + registerDataEntries(context); + registerContractDefinition(context); + + context.getMonitor().info("Policy Extension for Policy Sample (contract-negotiation) initialized!"); + } + + private PolicyDefinition createAccessPolicy() { + + var usePermission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance().type("USE").build()) + .build(); + + return PolicyDefinition.Builder.newInstance() + .id("use") + .policy(Policy.Builder.newInstance() + .permission(usePermission) + .build()) + .build(); + } + + private PolicyDefinition createContractPolicy(ServiceExtensionContext context) { + var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); + var notBeforeConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.GT) + .rightExpression(new LiteralExpression(startDate)) + .build(); + + var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T00:00:00.000+02:00"); + var notAfterConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(policyTimeKey)) + .operator(Operator.LT) + .rightExpression(new LiteralExpression(endDate)) + .build(); + + + var permission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance().type("USE").build()) + .constraint(notBeforeConstraint) + .constraint(notAfterConstraint) + .build(); + + + return PolicyDefinition.Builder.newInstance() + .id("use-time-restricted") + .policy(Policy.Builder.newInstance() + .permission(permission) + .build()) + .build(); + } + + + private void registerDataEntries(ServiceExtensionContext context) { + var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); + var assetPath = Path.of(assetPathSetting); + + var dataAddress = DataAddress.Builder.newInstance() + .property("type", "File") + .property("path", assetPath.getParent().toString()) + .property("filename", assetPath.getFileName().toString()) + .build(); + + var assetId = "test-document"; + var asset = Asset.Builder.newInstance().id(assetId).build(); + + assetIndex.accept(asset, dataAddress); + } + + private void registerContractDefinition(ServiceExtensionContext context) { + var accessPolicy = createAccessPolicy(); + policyStore.save(accessPolicy); + + var contractPolicy = createContractPolicy(context); + policyStore.save(contractPolicy); + + var contractDefinition = ContractDefinition.Builder.newInstance() + .id("1") + .accessPolicyId(accessPolicy.getUid()) + .contractPolicyId(contractPolicy.getUid()) + .selectorExpression(AssetSelectorExpression.Builder.newInstance() + .whenEquals(Asset.PROPERTY_ID, "test-document") + .build()) + .validity(31536000) + .build(); + contractDefinitionStore.save(contractDefinition); + } + +} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java new file mode 100644 index 00000000..055edbdd --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java @@ -0,0 +1,48 @@ +package org.eclipse.edc.sample.extension.policy; + + +import org.eclipse.edc.policy.engine.spi.AtomicConstraintFunction; +import org.eclipse.edc.policy.engine.spi.PolicyContext; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class TimeIntervalFunction implements AtomicConstraintFunction { + + private Monitor monitor; + + public TimeIntervalFunction(Monitor monitor) { + this.monitor = monitor; + } + + + + @Override + public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + Date date; + try { + date = sdf.parse((String) rightValue); + } catch (ParseException e) { + monitor.severe("Failed to parse right value of constraint to date."); + return false; + } + + switch (operator) { + case LT: var isBefore = new Date().before(date); + monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date"); + return isBefore; + case GT: var isAfter = new Date().after(date); + monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date"); + return isAfter; + default: return false; + } + } +} + + diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..1b26f404 --- /dev/null +++ b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.extension.policy.PolicyFunctionsExtension \ No newline at end of file From 82c97a75efcd49510bda6e5bf7bbfe06afabac0a Mon Sep 17 00:00:00 2001 From: Ronja Quensel Date: Mon, 11 Sep 2023 14:07:28 +0200 Subject: [PATCH 24/35] chore: update policy 1 README --- .../policy-01-contract-negotiation/README.md | 147 ------------------ .../contractoffer.json | 70 --------- .../build.gradle.kts | 34 ---- .../config.properties | 14 -- .../build.gradle.kts | 14 -- .../policy/PolicyFunctionsExtension.java | 143 ----------------- .../policy/TimeIntervalFunction.java | 48 ------ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - 8 files changed, 471 deletions(-) delete mode 100644 policy/policy-01-contract-negotiation/README.md delete mode 100644 policy/policy-01-contract-negotiation/contractoffer.json delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java delete mode 100644 policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/policy/policy-01-contract-negotiation/README.md b/policy/policy-01-contract-negotiation/README.md deleted file mode 100644 index 5778a1da..00000000 --- a/policy/policy-01-contract-negotiation/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Create a policy function - -This sample shows how a custom function for policy evaluation can be implemented and registered. For simplicity, -we will use just one connector, which initiates a contract negotiation with itself. The sample contains two modules: - -- `policy-contract-negotiation-connector`: contains the connector configuration -- `policy-contract-negotiation-policy-functions`: provides the function for policy evaluation - -## Creating and registering the function - -The [`TimeIntervalFunction`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java) -implements the `AtomicConstraintFunction` interface, which contains a single method for evaluating a constraint. -In that method, the `operator` and `right value` of the constraint can be used for evaluation. In this example, we -want to check whether the evaluation happens within a specified time interval. So depending on whether the operator is -`GT` or `LT`, we check if the current date is before or after the date specified in the `right value`. - -```java -@Override -public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { - var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - - Date date; - try { - date = sdf.parse((String) rightValue); - } catch (ParseException e) { - monitor.severe("Failed to parse right value of constraint to date."); - return false; - } - - switch (operator) { - case LT: var isBefore = new Date().before(date); - monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date."); - return isBefore; - case GT: var isAfter = new Date().after(date); - monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date."); - return isAfter; - default: return false; - } -} -``` - -Next, we have to register our function with the `PolicyEngine` and bind the desired action as well as the key used to -register our function to the desired scopes using the `RuleBindingRegistry`. This is done in the -[`PolicyFunctionsExtension`](policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java): - -```java -private final String policyTimeKey = "POLICY_EVALUATION_TIME"; - -//... - -@Override -public void initialize(ServiceExtensionContext context) { - //... - - ruleBindingRegistry.bind("USE", ALL_SCOPES); - ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); - policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); - - //... -} -``` - -## Defining the policy - -Next to registering the function and creating the binding, the `PolicyFunctionsExtension` also creates an asset, -a policy and a contract definition linking the former two. The asset is the same one that is used in -sample [`transfer-01-file-transfer`](transfer/transfer-01-file-transfer/README.md), but this time a different policy is created. - -We start by creating the constraints, where we define that the policy evaluation time must be within a certain -time interval. We read the start and end dates for the interval from the settings. Then, we create two constraints: -one specifying that the evaluation time should be after our defined start date, and one specifying that the evaluation -time should be before our defined end date. **We need to set the constraints' left operands to the same key -that we previously used to register our function. Otherwise, the function will not be used to evaluate these -constraints.** - -```java -var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); -var notBeforeConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.GT) - .rightExpression(new LiteralExpression(startDate)) - .build(); - -var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T23:59:00.000+02:00"); -var notAfterConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.LT) - .rightExpression(new LiteralExpression(endDate)) - .build(); -``` - -Then, we create a `Permission` with action type `USE` and the two constraints. We use this permission to create and -store a policy. And last, we create the `ContractDefinition`. For the access policy, we use the same use-policy that is -used in sample `transfer:transfer-01-file-transfer`, and for the contract policy, we use the previously created policy with the time -interval restriction. We set the `AssetSelectorExpression` so that the contract definition is valid for our asset. - -## How to run the sample - -To see the policy enforcement working, this sample should be run twice with different configurations. - -### Configuration - -Choose one of the following configurations. Depending on which configuration you choose, the contract negotiation -will either be confirmed or declined. - -#### 1. Policy fulfilled - -Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is within -the defined interval. - -```properties -edc.samples.uc.constraint.date.start=2023-01-01T00:00:00.000+02:00 -edc.samples.uc.constraint.date.end=2023-12-31T23:59:00.000+02:00 -``` - -#### 2. Policy not fulfilled - -Set the start and end date in the [`config.properties`](policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties) so that the current time is -**not** within the defined interval. - -```properties -edc.samples.policy-01.constraint.date.start=2022-01-01T00:00:00.000+02:00 -edc.samples.policy-01.constraint.date.end=2022-12-31T23:59:00.000+02:00 -``` - -### Run the sample - -First, build and run the connector for this sample: - -```shell -./gradlew policy:policy-01-contract-negotiation:policy-contract-negotiation-connector:build -java -Dedc.fs.config=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar -#for windows: -java -D"edc.fs.config"=policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties -jar policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build/libs/connector.jar -``` - -Next, initiate a contract negotiation. The request body is prepared in [`contractoffer.json`](policy/policy-01-contract-negotiation/contractoffer.json). -In that file, replace the dates for the constraints with the dates you used in the `config.properties` file and then -run: - -```shell -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @samples/uc-workshop/contractoffer.json "http://localhost:8182/api/v1/management/contractnegotiations" -``` - -Depending on the configuration you chose, you can see the contract negotiation either being confirmed or declined -in the connector logs. The logs also contain messages logged by the `TimeIntervalFunction`, which tell whether the -current date is before or after the start and end dates defined in the policy. diff --git a/policy/policy-01-contract-negotiation/contractoffer.json b/policy/policy-01-contract-negotiation/contractoffer.json deleted file mode 100644 index 9af961ad..00000000 --- a/policy/policy-01-contract-negotiation/contractoffer.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "connectorId": "provider", - "connectorAddress": "http://localhost:8282/api/v1/ids/data", - "protocol": "ids-multipart", - "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", - "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [ - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2023-01-01T00:00:00.000+02:00" - }, - "operator": "GT" - }, - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2023-12-31T00:00:00.000+02:00" - }, - "operator": "LT" - } - ], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } - } - } -} \ No newline at end of file diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts deleted file mode 100644 index a60d9218..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/build.gradle.kts +++ /dev/null @@ -1,34 +0,0 @@ -plugins { - `java-library` - id("application") - id("com.github.johnrengelman.shadow") version "7.1.2" -} - -val groupId: String by project -val edcVersion: String by project - -dependencies { - implementation("$groupId:control-plane-core:$edcVersion") - - implementation("$groupId:api-observability:$edcVersion") - - implementation("$groupId:configuration-filesystem:$edcVersion") - implementation("$groupId:iam-mock:$edcVersion") - - implementation("$groupId:auth-tokenbased:$edcVersion") - implementation("$groupId:management-api:$edcVersion") - - implementation("$groupId:ids:$edcVersion") - - implementation(project(":policy:policy-01-contract-negotiation:policy-contract-negotiation-policy-functions")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - exclude("**/pom.properties", "**/pom.xm") - mergeServiceFiles() - archiveFileName.set("connector.jar") -} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties b/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties deleted file mode 100644 index 4a9f967f..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties +++ /dev/null @@ -1,14 +0,0 @@ -web.http.port=8181 -web.http.path=/api - -web.http.management.port=8182 -web.http.management.path=/api/v1/management - -edc.api.auth.key=password -ids.webhook.address=http://localhost:8282 -edc.ids.id=urn:connector:provider - -edc.samples.policy-01.asset.path = /path/to/file - -edc.samples.policy-01.constraint.date.start=2023-01-01T00:00:00.000+02:00 -edc.samples.policy-01.constraint.date.end=2023-12-31T00:00:00.000+02:00 diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts deleted file mode 100644 index df01ec91..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/build.gradle.kts +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - `java-library` - id("application") -} - -val groupId: String by project -val edcVersion: String by project - -dependencies { - api("$groupId:data-plane-spi:$edcVersion") - - implementation("$groupId:control-plane-core:$edcVersion") - -} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java deleted file mode 100644 index a2dca9f6..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java +++ /dev/null @@ -1,143 +0,0 @@ -package org.eclipse.edc.sample.extension.policy; - -import org.eclipse.edc.connector.contract.spi.offer.store.ContractDefinitionStore; -import org.eclipse.edc.connector.contract.spi.types.offer.ContractDefinition; -import org.eclipse.edc.connector.policy.spi.PolicyDefinition; -import org.eclipse.edc.connector.policy.spi.store.PolicyDefinitionStore; -import org.eclipse.edc.policy.engine.spi.PolicyEngine; -import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; -import org.eclipse.edc.policy.model.*; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.asset.AssetIndex; -import org.eclipse.edc.spi.asset.AssetSelectorExpression; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.domain.DataAddress; -import org.eclipse.edc.spi.types.domain.asset.Asset; - -import java.nio.file.Path; - -import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; - -public class PolicyFunctionsExtension implements ServiceExtension { - private final String policyTimeKey = "POLICY_EVALUATION_TIME"; - private final String policyStartDateSetting = "edc.samples.policy-01.constraint.date.start"; - private final String policyEndDateSetting = "edc.samples.policy-01.constraint.date.end"; - private final String EDC_ASSET_PATH = "edc.samples.policy-01.asset.path"; - - @Inject - private RuleBindingRegistry ruleBindingRegistry; - - @Inject - private PolicyEngine policyEngine; - - @Inject - private PolicyDefinitionStore policyStore; - - @Inject - private ContractDefinitionStore contractDefinitionStore; - - @Inject - private AssetIndex assetIndex; - - @Override - public String name() { - return "Policy - contract-negotiation policies"; - } - - @Override - public void initialize(ServiceExtensionContext context) { - var monitor = context.getMonitor(); - - ruleBindingRegistry.bind("USE", ALL_SCOPES); - ruleBindingRegistry.bind(policyTimeKey, ALL_SCOPES); - policyEngine.registerFunction(ALL_SCOPES, Permission.class, policyTimeKey, new TimeIntervalFunction(monitor)); - - registerDataEntries(context); - registerContractDefinition(context); - - context.getMonitor().info("Policy Extension for Policy Sample (contract-negotiation) initialized!"); - } - - private PolicyDefinition createAccessPolicy() { - - var usePermission = Permission.Builder.newInstance() - .action(Action.Builder.newInstance().type("USE").build()) - .build(); - - return PolicyDefinition.Builder.newInstance() - .id("use") - .policy(Policy.Builder.newInstance() - .permission(usePermission) - .build()) - .build(); - } - - private PolicyDefinition createContractPolicy(ServiceExtensionContext context) { - var startDate = context.getSetting(policyStartDateSetting, "2023-01-01T00:00:00.000+02:00"); - var notBeforeConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.GT) - .rightExpression(new LiteralExpression(startDate)) - .build(); - - var endDate = context.getSetting(policyEndDateSetting, "2023-12-31T00:00:00.000+02:00"); - var notAfterConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyTimeKey)) - .operator(Operator.LT) - .rightExpression(new LiteralExpression(endDate)) - .build(); - - - var permission = Permission.Builder.newInstance() - .action(Action.Builder.newInstance().type("USE").build()) - .constraint(notBeforeConstraint) - .constraint(notAfterConstraint) - .build(); - - - return PolicyDefinition.Builder.newInstance() - .id("use-time-restricted") - .policy(Policy.Builder.newInstance() - .permission(permission) - .build()) - .build(); - } - - - private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, "/tmp/provider/test-document.txt"); - var assetPath = Path.of(assetPathSetting); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("path", assetPath.getParent().toString()) - .property("filename", assetPath.getFileName().toString()) - .build(); - - var assetId = "test-document"; - var asset = Asset.Builder.newInstance().id(assetId).build(); - - assetIndex.accept(asset, dataAddress); - } - - private void registerContractDefinition(ServiceExtensionContext context) { - var accessPolicy = createAccessPolicy(); - policyStore.save(accessPolicy); - - var contractPolicy = createContractPolicy(context); - policyStore.save(contractPolicy); - - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(accessPolicy.getUid()) - .contractPolicyId(contractPolicy.getUid()) - .selectorExpression(AssetSelectorExpression.Builder.newInstance() - .whenEquals(Asset.PROPERTY_ID, "test-document") - .build()) - .validity(31536000) - .build(); - contractDefinitionStore.save(contractDefinition); - } - -} diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java deleted file mode 100644 index 055edbdd..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/TimeIntervalFunction.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.eclipse.edc.sample.extension.policy; - - -import org.eclipse.edc.policy.engine.spi.AtomicConstraintFunction; -import org.eclipse.edc.policy.engine.spi.PolicyContext; -import org.eclipse.edc.policy.model.Operator; -import org.eclipse.edc.policy.model.Permission; -import org.eclipse.edc.spi.monitor.Monitor; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -public class TimeIntervalFunction implements AtomicConstraintFunction { - - private Monitor monitor; - - public TimeIntervalFunction(Monitor monitor) { - this.monitor = monitor; - } - - - - @Override - public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { - var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - - Date date; - try { - date = sdf.parse((String) rightValue); - } catch (ParseException e) { - monitor.severe("Failed to parse right value of constraint to date."); - return false; - } - - switch (operator) { - case LT: var isBefore = new Date().before(date); - monitor.info("Current date is " + (isBefore ? "before" : "after") + " desired end date"); - return isBefore; - case GT: var isAfter = new Date().after(date); - monitor.info("Current date is " + (isAfter ? "after" : "before") + " desired start date"); - return isAfter; - default: return false; - } - } -} - - diff --git a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 1b26f404..00000000 --- a/policy/policy-01-contract-negotiation/policy-contract-negotiation-policy-functions/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.sample.extension.policy.PolicyFunctionsExtension \ No newline at end of file From 934b8ff92ead4754fd8f0a1e2102636aa7162783 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 14 Nov 2023 14:10:52 +0100 Subject: [PATCH 25/35] update: Policy Sample update --- gradle/libs.versions.toml | 6 +++--- policy/policy-01-policy-enforcement/README.md | 4 ++-- .../contractoffer.json | 12 +++++------ .../build.gradle.kts | 5 +++-- .../config.properties | 11 ++++++++++ .../BasicPolicySampleConfirmContractTest.java | 8 ++++---- .../BasicPolicySampleDeclineContractTest.java | 8 ++++---- .../BasicPolicySampleTestCommon.java | 4 ++-- .../build.gradle.kts | 3 ++- .../config.properties | 19 +++++++++--------- .../policy/PolicyFunctionsExtension.java | 20 +++++++++++-------- 11 files changed, 58 insertions(+), 42 deletions(-) create mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-consumer/config.properties diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 409d1069..8c71026a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,13 +6,13 @@ assertj = "3.24.2" awaitility = "4.2.0" edc = "0.3.1" jakarta-json = "2.0.1" -junit-pioneer = "2.1.0" +junit-pioneer = "2.2.0" jupiter = "5.10.1" okhttp-mockwebserver = "5.0.0-alpha.11" openTelemetry = "1.18.0" restAssured = "5.3.2" rsApi = "3.1.0" -testcontainers = "1.19.1" +testcontainers = "1.19.2" kafkaClients = "3.6.0" [libraries] @@ -74,7 +74,7 @@ testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", ve kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } -opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.31.0" +opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.32.0" [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/policy/policy-01-policy-enforcement/README.md b/policy/policy-01-policy-enforcement/README.md index f45c571f..fcfe4d14 100644 --- a/policy/policy-01-policy-enforcement/README.md +++ b/policy/policy-01-policy-enforcement/README.md @@ -224,7 +224,7 @@ which protocol to use and which offer we want to negotiate. The request body is [contractoffer.json](./contractoffer.json). To start the negotiation, run the following command: ```bash -curl -X POST -H "Content-Type: application/json" -d @policy/policy-01-policy-enforcement/contractoffer.json "http://localhost:9192/api/management/v2/contractnegotiations" +curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @policy/policy-01-policy-enforcement/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" ``` You'll get back a UUID. This is the ID of the contract negotiation process which is being asynchronously executed @@ -236,7 +236,7 @@ Using the ID received in the previous step, we can now view the state of the neg of the consumer's data management API: ```bash -curl -X GET "http://localhost:9192/api/management/v2/contractnegotiations/" +curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/" ``` In the response we'll get a description of the negotiation, similar to the following: diff --git a/policy/policy-01-policy-enforcement/contractoffer.json b/policy/policy-01-policy-enforcement/contractoffer.json index 6fcf1a0c..4313071e 100644 --- a/policy/policy-01-policy-enforcement/contractoffer.json +++ b/policy/policy-01-policy-enforcement/contractoffer.json @@ -7,18 +7,18 @@ "connectorId": "provider", "consumerId": "consumer", "providerId": "provider", - "connectorAddress": "http://localhost:8282/api/protocol", + "connectorAddress": "http://localhost:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "", - "assetId": "", + "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", + "assetId": "test-document", "policy": { - "@id": "", + "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", "@type": "set", "odrl:permission": [], "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "" + "odrl:target": "test-document" } } -} \ No newline at end of file +} diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts index 40513d2e..ec16b298 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/build.gradle.kts @@ -11,8 +11,9 @@ dependencies { implementation(libs.edc.iam.mock) implementation(libs.edc.management.api) + implementation(libs.edc.data.plane.selector.core) - implementation(libs.edc.ids) + implementation(libs.edc.dsp) } application { @@ -20,7 +21,7 @@ application { } tasks.withType { - exclude("**/pom.properties", "**/pom.xm") + //exclude("**/pom.properties", "**/pom.xm") mergeServiceFiles() archiveFileName.set("consumer.jar") } diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-consumer/config.properties b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/config.properties new file mode 100644 index 00000000..1655aa4f --- /dev/null +++ b/policy/policy-01-policy-enforcement/policy-enforcement-consumer/config.properties @@ -0,0 +1,11 @@ +web.http.port=9191 +web.http.path=/api +web.http.management.port=9192 +web.http.management.path=/management +web.http.protocol.port=9292 +web.http.protocol.path=/protocol + +edc.api.auth.key=password +edc.dsp.callback.address=http://localhost:9292/protocol +edc.participant.id=consumer +edc.ids.id=urn:connector:consumer diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java index aedd3eff..91c4c735 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java +++ b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java @@ -31,13 +31,13 @@ @EndToEndTest public class BasicPolicySampleConfirmContractTest { - static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties"; - static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-contract-negotiation/contractoffer.json"; + static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties"; + static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-policy-enforcement/contractoffer.json"; @RegisterExtension static EdcRuntimeExtension connector = new EdcRuntimeExtension( - ":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector", - "connector", + ":policy:policy-01-policy-enforcement:policy-enforcement-provider", + "provider", Map.of( "edc.fs.config", getFileFromRelativePath(CONNECTOR_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() ) diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java index 1501c239..d54dd534 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java +++ b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java @@ -30,13 +30,13 @@ */ @EndToEndTest public class BasicPolicySampleDeclineContractTest { - static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-connector/config.properties"; - static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-contract-negotiation/policy-contract-negotiation-integration-tests/src/testFixtures/resources/contractoffer.json"; + static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties"; + static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-policy-enforcement/contractoffer.json"; @RegisterExtension static EdcRuntimeExtension connector = new EdcRuntimeExtension( - ":policy:policy-01-contract-negotiation:policy-contract-negotiation-connector", - "connector", + ":policy:policy-01-policy-enforcement:policy-enforcement-provider", + "provider", Map.of( // Override 'edc.samples.policy-01.constraint.date.start' & 'edc.samples.policy-01.constraint.date.end' implicitly set via property 'edc.fs.config'. "edc.samples.policy-01.constraint.date.start", "2022-01-01T00:00:00.000+02:00", diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java index 8fcb92bc..48afa38f 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java +++ b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java @@ -35,8 +35,8 @@ public class BasicPolicySampleTestCommon { //region constant test settings - static final String INITIATE_CONTRACT_NEGOTIATION_URI = "http://localhost:8182/api/v1/management/contractnegotiations"; - static final String LOOK_UP_CONTRACT_AGREEMENT_URI = "http://localhost:8182/api/v1/management/contractnegotiations/{id}"; + private static final String INITIATE_CONTRACT_NEGOTIATION_URI = "http://localhost:9192/management/v2/contractnegotiations"; + private static final String LOOK_UP_CONTRACT_AGREEMENT_URI = "http://localhost:9192/management/v2/contractnegotiations/{id}"; static final String API_KEY_HEADER_KEY = "X-Api-Key"; static final String API_KEY_HEADER_VALUE = "password"; //endregion diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts index 9d6023b2..112974c6 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts +++ b/policy/policy-01-policy-enforcement/policy-enforcement-provider/build.gradle.kts @@ -10,6 +10,7 @@ dependencies { implementation(libs.edc.configuration.filesystem) implementation(libs.edc.iam.mock) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.dsp) @@ -21,7 +22,7 @@ application { } tasks.withType { - exclude("**/pom.properties", "**/pom.xm") + // exclude("**/pom.properties", "**/pom.xm") mergeServiceFiles() archiveFileName.set("provider.jar") } diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties b/policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties index 4a9f967f..7ce71ce0 100644 --- a/policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties +++ b/policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties @@ -1,14 +1,13 @@ web.http.port=8181 web.http.path=/api - web.http.management.port=8182 -web.http.management.path=/api/v1/management - -edc.api.auth.key=password -ids.webhook.address=http://localhost:8282 +web.http.management.path=/management +web.http.protocol.port=8282 +web.http.protocol.path=/protocol +web.http.control.port=8283 +web.http.control.path=/control +edc.samples.policy-01.asset.path=/path/to/file +edc.dsp.callback.address=http://localhost:8282/protocol +edc.participant.id=provider edc.ids.id=urn:connector:provider - -edc.samples.policy-01.asset.path = /path/to/file - -edc.samples.policy-01.constraint.date.start=2023-01-01T00:00:00.000+02:00 -edc.samples.policy-01.constraint.date.end=2023-12-31T00:00:00.000+02:00 +edc.control.endpoint=http://localhost:8283/control diff --git a/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java index d18baca3..59372d9f 100644 --- a/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java +++ b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -29,6 +29,7 @@ import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.asset.AssetSelectorExpression; +import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.domain.DataAddress; @@ -135,26 +136,29 @@ private void registerDataEntries(ServiceExtensionContext context) { .build(); var assetId = "test-document"; - var asset = Asset.Builder.newInstance().id(assetId).build(); + var asset = Asset.Builder.newInstance() + .id(assetId) + .dataAddress(dataAddress) + .build(); + + + assetIndex.create(asset); - assetIndex.accept(asset, dataAddress); } + private void registerContractDefinition(ServiceExtensionContext context) { var accessPolicy = createAccessPolicy(); - policyStore.save(accessPolicy); + policyStore.create(accessPolicy); var contractPolicy = createContractPolicy(context); - policyStore.save(contractPolicy); + policyStore.create(contractPolicy); var contractDefinition = ContractDefinition.Builder.newInstance() .id("1") .accessPolicyId(accessPolicy.getUid()) .contractPolicyId(contractPolicy.getUid()) - .selectorExpression(AssetSelectorExpression.Builder.newInstance() - .whenEquals(Asset.PROPERTY_ID, "test-document") - .build()) - .validity(31536000) + .assetsSelectorCriterion(Criterion.criterion(Asset.PROPERTY_ID,"=", "test-document")) .build(); contractDefinitionStore.save(contractDefinition); } From 9783bcc74d66d31ffabcde1a756293074cac6a21 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 14 Nov 2023 14:10:52 +0100 Subject: [PATCH 26/35] update: Policy Sample update --- .github/workflows/verify.yaml | 12 - README.md | 8 + gradle/libs.versions.toml | 1 + settings.gradle.kts | 13 +- system-tests/build.gradle.kts | 3 +- .../samples/common/FileTransferCommon.java | 181 -------------- .../streaming/KafkaSaslContainer.java | 37 +++ .../Streaming03KafkaToKafkaTest.java | 233 ++++++++++++++++++ .../streaming-03-kafka-broker/1-asset.json | 11 + .../2-policy-definition.json | 10 + .../3-contract-definition.json | 7 + .../4-get-dataset.json | 7 + .../streaming-03-kafka-broker/6-transfer.json | 14 ++ .../streaming-03-kafka-broker/README.md | 205 +++++++++++++++ .../kafka-config/admin.properties | 3 + .../streaming-03-kafka-broker/kafka.env | 10 + .../kafka_broker_jaas.conf | 7 + .../streaming-03-runtime/build.gradle.kts | 48 ++++ .../streaming-03-runtime/consumer.properties | 11 + .../streaming-03-runtime/provider.properties | 13 + .../edc/samples/streaming/KafkaExtension.java | 40 +++ .../KafkaToKafkaDataFlowController.java | 55 +++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + transfer/transfer-00-prerequisites/README.md | 4 +- transfer/transfer-01-negotiation/README.md | 12 +- transfer/transfer-02-consumer-pull/README.md | 4 +- transfer/transfer-04-event-consumer/README.md | 6 +- 27 files changed, 740 insertions(+), 216 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/1-asset.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/6-transfer.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/README.md create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka.env create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b6ee42e3..e25e91d6 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -15,18 +15,6 @@ concurrency: cancel-in-progress: true jobs: - Checkstyle: - permissions: - id-token: write - checks: write - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-build - - - name: Run Checkstyle - run: ./gradlew checkstyleMain checkstyleTest Build: runs-on: ubuntu-latest diff --git a/README.md b/README.md index f33195c5..b3b6f4d6 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. +### [Policy](./policy/README.md) + +These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what +configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide +custom code for their enforcement. + +All policy samples are located in the `policy` directory. + ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8c71026a..fcc215cc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -56,6 +56,7 @@ edc-runtime-metamodel = { module = "org.eclipse.edc:runtime-metamodel", version. edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } edc-transfer-process-api = { module = "org.eclipse.edc:transfer-process-api", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } +edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-transfer-spi = { module = "org.eclipse.edc:transfer-spi", version.ref = "edc" } edc-util = { module = "org.eclipse.edc:util", version.ref = "edc" } edc-vault-azure = { module = "org.eclipse.edc:vault-azure", version.ref = "edc" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 3a0f500b..298e6565 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,12 +28,10 @@ dependencyResolutionManagement { } } -// basic include(":basic:basic-01-basic-connector") include(":basic:basic-02-health-endpoint") include(":basic:basic-03-configuration") -// transfer include(":transfer:transfer-00-prerequisites:connector") include(":transfer:transfer-04-event-consumer:consumer-with-listener") @@ -43,23 +41,20 @@ include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider") include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") -include("transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") -include("transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") +include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") +include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") +include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") -// advanced include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") - -//policy include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") include(":policy:policy-01-policy-enforcement:policy-functions") include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") -// modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") -include("util:http-request-logger") +include(":util:http-request-logger") include(":system-tests") diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index f7f3e6a8..deb02448 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -39,7 +39,8 @@ dependencies { testCompileOnly(project(":transfer:transfer-04-event-consumer:listener")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) + testCompileOnly(project(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime")) testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-provider")) testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-consumer")) -} \ No newline at end of file +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java index 0e79a98b..1ff5f45e 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java @@ -15,10 +15,6 @@ package org.eclipse.edc.samples.common; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; import org.eclipse.edc.junit.testfixtures.TestUtils; import org.jetbrains.annotations.NotNull; @@ -26,52 +22,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; -import java.util.Map; - -import static io.restassured.RestAssured.given; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.COMPLETED; -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; /** * Encapsulates common settings, test steps, and helper methods for transfer samples */ public class FileTransferCommon { - static final ObjectMapper MAPPER = new ObjectMapper(); - - static final String MANAGEMENT_API_URL = "http://localhost:9192/management"; - static final String CONTRACT_OFFER_FILE_PATH = "transfer/transfer-01-file-transfer/contractoffer.json"; - static final String TRANSFER_FILE_PATH = "transfer/transfer-01-file-transfer/filetransfer.json"; - static final String API_KEY_HEADER_KEY = "X-Api-Key"; - static final String API_KEY_HEADER_VALUE = "password"; - - final String sampleAssetFilePath; - final File sampleAssetFile; - final File destinationFile; - Duration timeout = Duration.ofSeconds(30); - Duration pollInterval = Duration.ofMillis(500); - - String contractNegotiationId; - String contractAgreementId; - - /** - * Creates a new {@code FileTransferSampleTestCommon} instance. - * - * @param sampleAssetFilePath Relative path starting from the root of the project to a file which will be read from for transfer. - * @param destinationFilePath Relative path starting from the root of the project where the transferred file will be written to. - */ - public FileTransferCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { - this.sampleAssetFilePath = sampleAssetFilePath; - sampleAssetFile = getFileFromRelativePath(sampleAssetFilePath); - - destinationFile = getFileFromRelativePath(destinationFilePath); - } - /** * Resolves a {@link File} instance from a relative path. */ @@ -93,141 +49,4 @@ public static String getFileContentFromRelativePath(String relativePath) { } } - /** - * Assert that prerequisites are fulfilled before running the test. - * This assertion checks only whether the file to be copied is not existing already. - */ - void assertTestPrerequisites() { - assertThat(destinationFile).doesNotExist(); - } - - /** - * Remove files created while running the tests. - * The copied file will be deleted. - */ - void cleanTemporaryTestFiles() { - destinationFile.delete(); - } - - /** - * Assert that the file to be copied exists at the expected location. - * This method waits a duration which is defined in {@link FileTransferCommon#timeout}. - */ - void assertDestinationFileContent() { - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() - -> assertThat(destinationFile).hasSameBinaryContentAs(sampleAssetFile)); - } - - /** - * Assert that the transfer process state on the consumer is completed. - */ - void assertTransferProcessStatusConsumerSide(String transferProcessId) { - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> { - var state = getTransferProcessState(transferProcessId); - - assertThat(state).isEqualTo(COMPLETED.name()); - }); - } - - /** - * Assert that a POST request to initiate a contract negotiation is successful. - * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"} - */ - void initiateContractNegotiation() { - initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - } - - - void initiateContractNegotiation(String contractOfferFilePath) { - contractNegotiationId = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) - .when() - .post(MANAGEMENT_API_URL + "/v2/contractnegotiations") - .then() - .statusCode(HttpStatus.SC_OK) - .body("@id", not(emptyString())) - .extract() - .jsonPath() - .get("@id"); - } - - public String getTransferProcessState(String processId) { - return given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(String.format("%s/%s", MANAGEMENT_API_URL + "/v2/transferprocesses", processId)) - .then() - .statusCode(HttpStatus.SC_OK) - .extract().body().jsonPath().getString("'edc:state'"); - } - - /** - * Assert that a GET request to look up a contract agreement is successful. - * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}"} - */ - void lookUpContractAgreementId() { - // Wait for transfer to be completed. - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> contractAgreementId = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(MANAGEMENT_API_URL + "/v2/contractnegotiations/{id}", contractNegotiationId) - .then() - .statusCode(HttpStatus.SC_OK) - .body("'edc:state'", equalTo("FINALIZED")) - .body("'edc:contractAgreementId'", not(emptyString())) - .extract().body().jsonPath().getString("'edc:contractAgreementId'") - ); - } - - /** - * Assert that a POST request to initiate transfer process is successful. - * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses"} - * - * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferCommon#TRANSFER_FILE_PATH}. - */ - String requestTransferFile(String transferFilePath) throws IOException { - var transferJsonFile = getFileFromRelativePath(transferFilePath); - var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId); - - var jsonPath = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(requestBody) - .when() - .post(MANAGEMENT_API_URL + "/v2/transferprocesses") - .then() - .log().ifError() - .statusCode(HttpStatus.SC_OK) - .body("@id", not(emptyString())) - .extract() - .jsonPath(); - - var transferProcessId = jsonPath.getString("@id"); - - assertThat(transferProcessId).isNotEmpty(); - - return transferProcessId; - } - - String requestTransferFile() throws IOException { - return requestTransferFile(TRANSFER_FILE_PATH); - } - - /** - * Reads a transfer request file with changed value for contract agreement ID and file destination path. - * - * @param transferJsonFile A {@link File} instance pointing to a JSON transfer request file. - * @param contractAgreementId This string containing a UUID will be used as value for the contract agreement ID. - * @return An instance of {@link DataRequest} with changed values for contract agreement ID and file destination path. - * @throws IOException Thrown if there was an error accessing the file given in transferJsonFile. - */ - Map readAndUpdateDataRequestFromJsonFile(@NotNull File transferJsonFile, @NotNull String contractAgreementId) throws IOException { - var fileString = Files.readString(transferJsonFile.toPath()) - .replace("{path to destination file}", destinationFile.getAbsolutePath()) - .replace("{agreement ID}", contractAgreementId); - - return MAPPER.readValue(fileString, Map.class); - } } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java new file mode 100644 index 00000000..c2df6d60 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java @@ -0,0 +1,37 @@ +package org.eclipse.edc.samples.transfer.streaming; + +import org.jetbrains.annotations.NotNull; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Extension of the {@link KafkaContainer} that permits to set the SASL_PLAINTEXT security protocol + */ +public class KafkaSaslContainer extends KafkaContainer { + + private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.5.2"; + private final File envFile; + + public KafkaSaslContainer(@NotNull File envFile) { + super(DockerImageName.parse(KAFKA_IMAGE_NAME)); + this.withKraft(); + this.envFile = envFile; + } + + @Override + protected void configureKraft() { + super.configureKraft(); + try { + Files.readAllLines(envFile.toPath()) + .stream().map(it -> it.split("=", 2)) + .forEach(it -> this.withEnv(it[0], it[1])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java new file mode 100644 index 00000000..c12fdaee --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial test implementation for sample + * + */ + +package org.eclipse.edc.samples.transfer.streaming; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.Json; +import okhttp3.mockwebserver.MockWebServer; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.eclipse.edc.spi.types.domain.edr.EndpointDataReference; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +import static java.time.Duration.ZERO; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; +import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; + +@Testcontainers +@EndToEndTest +public class Streaming03KafkaToKafkaTest { + + private static final String TOPIC = "topic-" + UUID.randomUUID(); + private static final String SAMPLE_NAME = "streaming-03-kafka-broker"; + private static final String RUNTIME_NAME = "streaming-03-runtime"; + private static final Path SAMPLE_FOLDER = Path.of("transfer", "streaming", SAMPLE_NAME); + private static final Path RUNTIME_PATH = SAMPLE_FOLDER.resolve(RUNTIME_NAME); + private static final Duration TIMEOUT = Duration.ofSeconds(60); + private static final Participant PROVIDER = Participant.Builder.newInstance() + .name("provider") + .id("provider") + .managementEndpoint(new Participant.Endpoint(URI.create("http://localhost:18181/management"))) + .protocolEndpoint(new Participant.Endpoint(URI.create("http://localhost:18182/protocol"))) + .controlEndpoint(new Participant.Endpoint(URI.create("http://localhost:18183/control"))) + .build(); + private static final Participant CONSUMER = Participant.Builder.newInstance() + .name("consumer") + .id("consumer") + .managementEndpoint(new Participant.Endpoint(URI.create("http://localhost:28181/management"))) + .protocolEndpoint(new Participant.Endpoint(URI.create("http://localhost:28182/protocol"))) + .controlEndpoint(new Participant.Endpoint(URI.create("http://localhost:28183/control"))) + .build(); + private static final String GROUP_ID = "group_id"; + + @Container + static KafkaContainer kafkaContainer = new KafkaSaslContainer(getFileFromRelativePath(SAMPLE_FOLDER.resolve("kafka.env").toString())) + .withLogConsumer(frame -> System.out.print(frame.getUtf8String())); + + @RegisterExtension + static EdcRuntimeExtension providerConnector = new EdcRuntimeExtension( + ":transfer:streaming:%s:%s".formatted(SAMPLE_NAME, RUNTIME_NAME), + "provider", + Map.of( + "edc.fs.config", + getFileFromRelativePath(RUNTIME_PATH.resolve("provider.properties").toString()) + .getAbsolutePath() + ) + ); + + @RegisterExtension + static EdcRuntimeExtension consumerConnector = new EdcRuntimeExtension( + ":transfer:streaming:%s:%s".formatted(SAMPLE_NAME, RUNTIME_NAME), + "consumer", + Map.of( + "edc.fs.config", + getFileFromRelativePath(RUNTIME_PATH.resolve("consumer.properties").toString()) + .getAbsolutePath() + ) + ); + + private final int httpReceiverPort = TestUtils.getFreePort(); + private final MockWebServer edrReceiverServer = new MockWebServer(); + + @BeforeEach + void setUp() throws IOException { + edrReceiverServer.start(httpReceiverPort); + } + + @Test + void streamData() throws InterruptedException, JsonProcessingException { + createAcls( + userCanAccess("User:alice", ResourceType.TOPIC, TOPIC), + userCanAccess("User:alice", ResourceType.GROUP, GROUP_ID) + ); + + PROVIDER.createAsset(getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("1-asset.json").toString()) + .replace("{{bootstrap.servers}}", kafkaContainer.getBootstrapServers()) + .replace("{{topic}}", TOPIC)); + PROVIDER.createPolicyDefinition(getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("2-policy-definition.json").toString())); + PROVIDER.createContractDefinition( + getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("3-contract-definition.json").toString())); + + var destination = Json.createObjectBuilder() + .add("type", "Kafka") + .build(); + + var transferProcessPrivateProperties = Json.createObjectBuilder() + .add("receiverHttpEndpoint", "http://localhost:" + httpReceiverPort) + .build(); + var transferProcessId = CONSUMER.requestAsset(PROVIDER, "kafka-stream-asset", transferProcessPrivateProperties, destination); + + await().atMost(TIMEOUT).untilAsserted(() -> { + var state = CONSUMER.getTransferProcessState(transferProcessId); + assertThat(state).isEqualTo(STARTED.name()); + }); + + var producer = createKafkaProducer(); + var message = "message"; + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> producer + .send(new ProducerRecord<>(TOPIC, "key", message)), 0L, 100L, MICROSECONDS); + + var endpointDataReference = readEndpointDataReference(); + + try (var clientConsumer = createKafkaConsumer(endpointDataReference.getEndpoint(), endpointDataReference.getAuthKey(), endpointDataReference.getAuthCode())) { + clientConsumer.subscribe(List.of(endpointDataReference.getProperties().get(EDC_NAMESPACE + "topic").toString())); + + await().atMost(TIMEOUT).untilAsserted(() -> { + var records = clientConsumer.poll(ZERO); + assertThat(records.isEmpty()).isFalse(); + records.records(TOPIC).forEach(record -> assertThat(record.value()).isEqualTo(message)); + }); + } + + producer.close(); + } + + private EndpointDataReference readEndpointDataReference() throws InterruptedException, JsonProcessingException { + var request = edrReceiverServer.takeRequest(TIMEOUT.getSeconds(), SECONDS); + var body = request.getBody().readString(Charset.defaultCharset()); + return new ObjectMapper().readValue(body, EndpointDataReference.class); + } + + private AclBinding userCanAccess(String principal, ResourceType resourceType, String resourceName) { + var pattern = new ResourcePattern(resourceType, resourceName, PatternType.LITERAL); + var entry = new AccessControlEntry(principal, "*", AclOperation.READ, AclPermissionType.ALLOW); + return new AclBinding(pattern, entry); + } + + private void createAcls(AclBinding... bindings) { + var adminProperties = new Properties(); + adminProperties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + adminProperties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + adminProperties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin-secret\";"); + adminProperties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + try (var adminClient = AdminClient.create(adminProperties)) { + adminClient.createAcls(Arrays.stream(bindings).toList()).all().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private Producer createKafkaProducer() { + var props = new Properties(); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin-secret\";"); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + return new KafkaProducer<>(props); + } + + private static Consumer createKafkaConsumer(@NotNull String endpoint, @Nullable String authKey, @Nullable String authCode) { + var props = new Properties(); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";".formatted(authKey, authCode)); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, endpoint); + props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + return new KafkaConsumer<>(props); + } + +} diff --git a/transfer/streaming/streaming-03-kafka-broker/1-asset.json b/transfer/streaming/streaming-03-kafka-broker/1-asset.json new file mode 100644 index 00000000..a4b7d626 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/1-asset.json @@ -0,0 +1,11 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@id": "kafka-stream-asset", + "properties": { + }, + "dataAddress": { + "type": "Kafka", + "kafka.bootstrap.servers": "{{bootstrap.servers}}", + "topic": "{{topic}}" + } +} diff --git a/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json b/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json new file mode 100644 index 00000000..4919c71a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json @@ -0,0 +1,10 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "no-constraint-policy", + "policy": { + "@type": "odrl:use" + } +} diff --git a/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json b/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json new file mode 100644 index 00000000..d424ec90 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@id": "contract-definition", + "accessPolicyId": "no-constraint-policy", + "contractPolicyId": "no-constraint-policy", + "assetsSelector": [] +} diff --git a/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json b/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json new file mode 100644 index 00000000..0ec57558 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@type": "DatasetRequest", + "@id": "kafka-stream-asset", + "counterPartyAddress": "http://localhost:18182/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/transfer/streaming/streaming-03-kafka-broker/6-transfer.json b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json new file mode 100644 index 00000000..0e2c7d38 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequest", + "dataDestination": { + "type": "Kafka" + }, + "protocol": "dataspace-protocol-http", + "assetId": "stream-asset", + "contractId": "{{contract-agreement-id}}", + "connectorId": "provider", + "connectorAddress": "http://localhost:18182/protocol" +} diff --git a/transfer/streaming/streaming-03-kafka-broker/README.md b/transfer/streaming/streaming-03-kafka-broker/README.md new file mode 100644 index 00000000..046c6c85 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/README.md @@ -0,0 +1,205 @@ +# Streaming KAFKA to KAFKA + +This sample demonstrates how to set up the EDC to stream messages through Kafka. +This code is only for demonstration purposes and should not be used in production. + +## Concept + +In this sample the Data-Plane is not used, the consumer will set up a kafka client to poll the messages from the broker +using some credentials obtained from the transfer process. + +The DataFlow is managed by the [KafkaToKafkaDataFlowController](streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java), +that on flow initialization creates an `EndpointDataReference` containing the credentials that the consumer would then use +to poll the messages. + +### Run + +Build the connector runtime, which will be used both for the provider and consumer: +```shell +./gradlew :transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime:build +``` + +Run the provider and the consumer, which must be started from different terminal shells: +```shell +# provider +export EDC_FS_CONFIG=transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties +java -jar transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build/libs/connector.jar + +#consumer +export EDC_FS_CONFIG=transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties +java -jar transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build/libs/connector.jar +``` + +### Start Kafka and configure ACLs + +Kafka will be started in [KRaft mode](https://developer.confluent.io/learn/kraft/), a single broker with `SASL_PLAINTEXT` +as security protocol ([see config](kafka.env)), there will be an `admin` user, responsible for setting up ACLs and producing +messages, and `alice`, that will be used by the consumer to consume the messages. + +Run the Kafka container: +```shell +docker run --rm --name=kafka-kraft -h kafka-kraft -p 9093:9093 \ + -v "$PWD/transfer/streaming/streaming-03-kafka-broker/kafka-config":/config \ + --env-file transfer/streaming/streaming-03-kafka-broker/kafka.env \ + -e KAFKA_NODE_ID=1 \ + -e KAFKA_LISTENERS='PLAINTEXT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://0.0.0.0:9094' \ + -e KAFKA_ADVERTISED_LISTENERS='PLAINTEXT://localhost:9093,BROKER://localhost:9092' \ + -e KAFKA_PROCESS_ROLES='broker,controller' \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS='1@localhost:9094' \ + -e KAFKA_INTER_BROKER_LISTENER_NAME='BROKER' \ + -e KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER' \ + -e KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1 \ + -e CLUSTER_ID='4L6g3nShT-eMCtK--X86sw' \ + confluentinc/cp-kafka:7.5.2 +``` + +Create the topic `kafka-stream-topic` +```shell +docker exec -it kafka-kraft /bin/kafka-topics \ + --topic kafka-stream-topic --create --partitions 1 --replication-factor 1 \ + --command-config=/config/admin.properties \ + --bootstrap-server localhost:9092 +``` + +To give `alice` read permissions on the topic we need to set up ACLs: +```shell +docker exec -it kafka-kraft /bin/kafka-acls --command-config /config/admin.properties \ + --bootstrap-server localhost:9093 \ + --add --allow-principal 'User:alice' \ + --topic kafka-stream-topic \ + --group group_id \ + --operation Read +``` + +### Register Asset, Policy Definition and Contract Definition on provider + +Then put values of `kafka.bootstrap.servers`, `maxDuration` and `topic` in the [1-asset.json](1-asset.json) file replacing +their placeholders this way: +```json + "dataAddress": { + "type": "Kafka", + "kafka.bootstrap.servers": "localhost:9093", + "topic": "kafka-stream-topic" + } +``` + +Then create the Asset, the Policy Definition and the Contract Definition with these three calls: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/1-asset.json -X POST "http://localhost:18181/management/v3/assets" +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json -X POST "http://localhost:18181/management/v2/policydefinitions" +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json -X POST "http://localhost:18181/management/v2/contractdefinitions" +``` + +### Negotiate the contract + +The typical flow requires fetching the catalog from the consumer side and using the contract offer to negotiate a contract. +However, in this sample case, we already have the provider asset (`"kafka-stream-asset"`) so we can get the related dataset +directly with this call: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json -X POST "http://localhost:28181/management/v2/catalog/dataset/request" -s | jq +``` + +The output will be something like: +```json +{ + "@id": "kafka-stream-asset", + "@type": "dcat:Dataset", + "odrl:hasPolicy": { + "@id": "Y29udHJhY3QtZGVmaW5pdGlvbg==:c3RyZWFtLWFzc2V0:NDlhYTUzZWEtMDUzMS00ZDkyLTg4Y2YtMGRjMTc4MmQ1NjY4", + "@type": "odrl:Set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "kafka-stream-asset" + }, + "dcat:distribution": { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpData" + }, + "dcat:accessService": "b24dfdbc-d17f-4d6e-9b5c-8fa71dacecfc" + }, + "edc:id": "kafka-stream-asset", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` + +With the `odrl:hasPolicy/@id` we can now replace it in the [negotiate-contract.json](5-negotiate-contract.json) file +and negotiate the contract: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/5-negotiate-contract.json -X POST "http://localhost:28181/management/v2/contractnegotiations" -s | jq +``` + +### Start the transfer + +First we need to set up the receiver server on the consumer side that will receive the EndpointDataReference containing +the address and credentials to connect to the broker and poll the messages from the topic. For this you'll need to open +another terminal shell and run: +```shell +./gradlew util:http-request-logger:build +HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar +``` +It will run on port 4000. + +At this point the contract agreement should already been issued, to verify that, please check the contract negotiation state with +this call, replacing `{{contract-negotiation-id}}` with the id returned by the negotiate contract call. +```shell +curl "http://localhost:28181/management/v2/contractnegotiations/{{contract-negotiation-id}}" -s | jq +``` + +If the `edc:contractAgreementId` is valued, it can be used to start the transfer, replacing it in the [6-transfer.json](6-transfer.json) +file to `{{contract-agreement-id}}` and then calling the connector with this command: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/6-transfer.json -X POST "http://localhost:28181/management/v2/transferprocesses" -s | jq +``` +> Note that the destination address is `localhost:4000`, this because is where our http server is listening. + +Let's wait until the transfer state is `STARTED` state executing this call, replacing to `{{transfer-process-id}}` the id returned +by the start transfer call: +```shell +curl "http://localhost:28181/management/v2/transferprocesses/{{transfer-process-id}}" -s | jq +``` + +### Consume events +Now in the console of the `http-request-logger` we started before, the `EndpointDataReference` should have appeared: +```json +{ + "id":"8c52a781-2588-4c9b-8c70-4e5ad428eea9", + "endpoint":"localhost:9093", + "authKey":"alice", + "authCode":"alice-secret", + "properties": { + "https://w3id.org/edc/v0.0.1/ns/topic":"kafka-stream-topic" + } +} +``` + +Using these information on the consumer side we can run a `kafka-console-consumer` with the data received to consume +messages from the topic: +```shell +docker exec -it kafka-kraft /bin/kafka-console-consumer --topic kafka-stream-topic \ + --bootstrap-server localhost:9093 \ + --consumer-property group.id=group_id \ + --consumer-property security.protocol=SASL_PLAINTEXT \ + --consumer-property sasl.mechanism=PLAIN \ + --consumer-property sasl.jaas.config='org.apache.kafka.common.security.plain.PlainLoginModule required username="alice" password="alice-secret";' +``` + +### Produce events + +In another shell we can put ourselves in the provider shoes and create messages from the producer shell: +```shell +docker exec -it kafka-kraft /bin/kafka-console-producer --topic kafka-stream-topic \ + --producer.config=/config/admin.properties \ + --bootstrap-server localhost:9093 +``` + +For every message created on the provider side we will see a message on the consumer side. diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties b/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties new file mode 100644 index 00000000..65d00847 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties @@ -0,0 +1,3 @@ +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret"; diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka.env b/transfer/streaming/streaming-03-kafka-broker/kafka.env new file mode 100644 index 00000000..f5faa1c0 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka.env @@ -0,0 +1,10 @@ +KAFKA_AUTHORIZER_CLASS_NAME=org.apache.kafka.metadata.authorizer.StandardAuthorizer +KAFKA_AUTO_CREATE_TOPICS_ENABLE=true +KAFKA_LISTENER_NAME_BROKER_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret"; +KAFKA_LISTENER_NAME_CONTROLLER_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret"; +KAFKA_LISTENER_NAME_PLAINTEXT_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret" user_alice="alice-secret"; +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=BROKER:SASL_PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT,CONTROLLER:SASL_PLAINTEXT +KAFKA_SASL_ENABLED_MECHANISMS=PLAIN +KAFKA_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN +KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN +KAFKA_SUPER_USERS=User:admin diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf b/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf new file mode 100644 index 00000000..12b5f8f9 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf @@ -0,0 +1,7 @@ +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="admin" + password="admin-secret" + user_admin="admin-secret" + user_alice="alice-secret"; +}; diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts new file mode 100644 index 00000000..5c6b5c0a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.api.observability) + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) + implementation(libs.edc.management.api) + implementation(libs.edc.dsp) + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.client) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.dynamic.receiver) + implementation(libs.edc.data.plane.spi) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + implementation(libs.edc.data.plane.kafka) + +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("connector.jar") +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties new file mode 100644 index 00000000..da471ae1 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties @@ -0,0 +1,11 @@ +web.http.port=28180 +web.http.path=/api +web.http.management.port=28181 +web.http.management.path=/management +web.http.protocol.port=28182 +web.http.protocol.path=/protocol +web.http.control.port=28183 +web.http.control.path=/control +edc.dsp.callback.address=http://localhost:28182/protocol +edc.participant.id=consumer +edc.receiver.http.dynamic.endpoint=http://localhost:4000/receiver diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties new file mode 100644 index 00000000..a06dcb9f --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties @@ -0,0 +1,13 @@ +web.http.port=18180 +web.http.path=/api +web.http.management.port=18181 +web.http.management.path=/management +web.http.protocol.port=18182 +web.http.protocol.path=/protocol +web.http.control.port=18183 +web.http.control.path=/control +edc.dsp.callback.address=http://localhost:18182/protocol +edc.participant.id=provider +edc.ids.id=urn:connector:provider +edc.dataplane.http.sink.partition.size=1 +edc.receiver.http.dynamic.endpoint=http://localhost/not/used/in/this/sample diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java new file mode 100644 index 00000000..7657cfb2 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.samples.streaming; + +import org.eclipse.edc.connector.transfer.spi.flow.DataFlowManager; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +/** + * Kafka Broker flow extension + */ +public class KafkaExtension implements ServiceExtension { + + @Override + public String name() { + return "Kafka stream extension"; + } + + @Inject + private DataFlowManager dataFlowManager; + + @Override + public void initialize(ServiceExtensionContext context) { + dataFlowManager.register(10, new KafkaToKafkaDataFlowController()); + } + +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java new file mode 100644 index 00000000..b1a11ecc --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.samples.streaming; + +import org.eclipse.edc.connector.transfer.spi.flow.DataFlowController; +import org.eclipse.edc.connector.transfer.spi.types.DataFlowResponse; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.dataplane.kafka.schema.KafkaDataAddressSchema; +import org.eclipse.edc.policy.model.Policy; +import org.eclipse.edc.spi.response.StatusResult; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.edr.EndpointDataReference; +import org.jetbrains.annotations.NotNull; + +import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; + +class KafkaToKafkaDataFlowController implements DataFlowController { + @Override + public boolean canHandle(TransferProcess transferProcess) { + return "Kafka".equals(transferProcess.getContentDataAddress().getType()) && "Kafka".equals(transferProcess.getDestinationType()); + } + + @Override + public @NotNull StatusResult initiateFlow(TransferProcess transferProcess, Policy policy) { + // static credentials, in a production case these should be created dynamically and an ACLs entry should be added + var username = "alice"; + var password = "alice-secret"; + + var contentDataAddress = transferProcess.getContentDataAddress(); + var kafkaDataAddress = DataAddress.Builder.newInstance() + .type(EndpointDataReference.EDR_SIMPLE_TYPE) + .property(EndpointDataReference.ID, transferProcess.getCorrelationId()) + .property(EndpointDataReference.ENDPOINT, contentDataAddress.getStringProperty("kafka.bootstrap.servers")) + .property(EndpointDataReference.AUTH_KEY, username) + .property(EndpointDataReference.AUTH_CODE, password) + .property(EDC_NAMESPACE + KafkaDataAddressSchema.TOPIC, contentDataAddress.getStringProperty(KafkaDataAddressSchema.TOPIC)) + .build(); + + return StatusResult.success(DataFlowResponse.Builder.newInstance().dataAddress(kafkaDataAddress).build()); + } + + // TODO: terminate data flow method will available in the next EDC version, it will permit to remove permissions to the user to access the topic +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..849cf54a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.samples.streaming.KafkaExtension diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md index d663a21b..b79ddad0 100644 --- a/transfer/transfer-00-prerequisites/README.md +++ b/transfer/transfer-00-prerequisites/README.md @@ -21,8 +21,8 @@ This sample will go through: ### 1. Build the connector -When we speak of a connector we actually mean a .jar file that is launched on a machine. -Before we can launch a connector we'll have to build the .jar file. +When we talk about a connector in the context of Eclipse Dataspace Components, we really mean a JAR file that runs on a machine. +Before we can run a connector, we need to build the JAR file. Execute this command in project root: diff --git a/transfer/transfer-01-negotiation/README.md b/transfer/transfer-01-negotiation/README.md index 885d9a6e..8ee5cd4a 100644 --- a/transfer/transfer-01-negotiation/README.md +++ b/transfer/transfer-01-negotiation/README.md @@ -73,10 +73,10 @@ curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \ ### 3. Create a contract definition on Provider -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition +To ensure an exchange between provider and consumer, the provider must create a contract offer for +the asset, on the basis of which a contract agreement can be negotiated. The contract definition associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies +catalog. In this case, the selection is empty, so every asset is attached to these policies. ```bash curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ @@ -98,7 +98,7 @@ Sample output: ### 4. How to fetch catalog on consumer side -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain +In order to request any data, the consumer must fetch the catalog from the provider, which contains all the contract offers available for negotiation. In our case, it will contain a single contract offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following request: @@ -179,7 +179,7 @@ looks as follows: reference Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counter offers in addition to just confirming or declining an offer. +send counteroffers in addition to just confirming or declining an offer. ```bash curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ @@ -224,7 +224,7 @@ Sample output: "edc:state": "FINALIZED", "edc:counterPartyAddress": "http://localhost:19194/protocol", "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", <--------- + "edc:contractAgreementId": "0b3150be-feaf-43bc-91e1-90f050de28bd", <--------- "@context": { "dct": "https://purl.org/dc/terms/", "edc": "https://w3id.org/edc/v0.0.1/ns/", diff --git a/transfer/transfer-02-consumer-pull/README.md b/transfer/transfer-02-consumer-pull/README.md index c1e41227..4de7b639 100644 --- a/transfer/transfer-02-consumer-pull/README.md +++ b/transfer/transfer-02-consumer-pull/README.md @@ -1,6 +1,6 @@ # Implement a simple "Consumer Pull" Http transfer flow -The purpose of this sample is to show a data exchange between 2 connectors, one representing the +The purpose of this sample is to show a data exchange between two connectors, one representing the data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find more details on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). @@ -25,7 +25,7 @@ order. ### 1. Start a http server -As a pre-requisite, you need to have a http server that runs on port 4000 and logs all the incoming requests, it will +As a pre-requisite, you need to have a logging webserver that runs on port 4000 and logs all the incoming requests, it will be mandatory to get the EndpointDataReference that will be used to get the data. ```bash diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md index 85952ce9..9780c9a4 100644 --- a/transfer/transfer-04-event-consumer/README.md +++ b/transfer/transfer-04-event-consumer/README.md @@ -50,13 +50,13 @@ public class TransferProcessStartedListener implements TransferProcessListener { ## Run the sample -Assuming your provider connector is still running, we can re-use the existing assets and contract definitions stored on +Assuming your provider connector and logging webserver are still running, we can re-use the existing assets and contract definitions stored on provider side. If not, set up your assets and contract definitions as described in the [Negotiation](../transfer-01-negotiation/README.md) chapter. ### 1. Build & launch the consumer with listener extension -This consumer connector is based on a different build file, hence a new jar file will be produced. +This consumer connector is based on a different build file, hence a new JAR file will be built. Make sure to terminate your current consumer connector from the previous chapters. That way we unblock the ports and can reuse the known configuration files and API calls. @@ -109,4 +109,4 @@ DEBUG 2023-10-16T09:29:46.27174 TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8 If you see the `TransferProcessStartedListener received STARTED event` log message, it means that your event consumer has been configured successfully. -[Next Chapter](../transfer-05-open-telemetry/README.md) +[Next Chapter](../transfer-05-file-transfer-cloud/README.md) From 9e9cfa29ee8d4766d59c01a952adbb1a026620de Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 14 Nov 2023 14:10:52 +0100 Subject: [PATCH 27/35] update: Policy Sample update --- .github/workflows/verify.yaml | 12 - README.md | 8 + gradle/libs.versions.toml | 1 + settings.gradle.kts | 13 +- system-tests/build.gradle.kts | 3 +- .../samples/common/FileTransferCommon.java | 181 -------------- .../streaming/KafkaSaslContainer.java | 37 +++ .../Streaming03KafkaToKafkaTest.java | 233 ++++++++++++++++++ .../streaming-03-kafka-broker/1-asset.json | 11 + .../2-policy-definition.json | 10 + .../3-contract-definition.json | 7 + .../4-get-dataset.json | 7 + .../streaming-03-kafka-broker/6-transfer.json | 14 ++ .../streaming-03-kafka-broker/README.md | 205 +++++++++++++++ .../kafka-config/admin.properties | 3 + .../streaming-03-kafka-broker/kafka.env | 10 + .../kafka_broker_jaas.conf | 7 + .../streaming-03-runtime/build.gradle.kts | 48 ++++ .../streaming-03-runtime/consumer.properties | 11 + .../streaming-03-runtime/provider.properties | 13 + .../edc/samples/streaming/KafkaExtension.java | 40 +++ .../KafkaToKafkaDataFlowController.java | 55 +++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + transfer/transfer-00-prerequisites/README.md | 4 +- transfer/transfer-01-negotiation/README.md | 12 +- transfer/transfer-02-consumer-pull/README.md | 4 +- transfer/transfer-04-event-consumer/README.md | 6 +- .../samples/util/HttpRequestLoggerServer.java | 2 +- 28 files changed, 741 insertions(+), 217 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/1-asset.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/6-transfer.json create mode 100644 transfer/streaming/streaming-03-kafka-broker/README.md create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka.env create mode 100644 transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java create mode 100644 transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b6ee42e3..e25e91d6 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -15,18 +15,6 @@ concurrency: cancel-in-progress: true jobs: - Checkstyle: - permissions: - id-token: write - checks: write - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-build - - - name: Run Checkstyle - run: ./gradlew checkstyleMain checkstyleTest Build: runs-on: ubuntu-latest diff --git a/README.md b/README.md index f33195c5..b3b6f4d6 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. +### [Policy](./policy/README.md) + +These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what +configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide +custom code for their enforcement. + +All policy samples are located in the `policy` directory. + ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8c71026a..fcc215cc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -56,6 +56,7 @@ edc-runtime-metamodel = { module = "org.eclipse.edc:runtime-metamodel", version. edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } edc-transfer-process-api = { module = "org.eclipse.edc:transfer-process-api", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } +edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-transfer-spi = { module = "org.eclipse.edc:transfer-spi", version.ref = "edc" } edc-util = { module = "org.eclipse.edc:util", version.ref = "edc" } edc-vault-azure = { module = "org.eclipse.edc:vault-azure", version.ref = "edc" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 3a0f500b..298e6565 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,12 +28,10 @@ dependencyResolutionManagement { } } -// basic include(":basic:basic-01-basic-connector") include(":basic:basic-02-health-endpoint") include(":basic:basic-03-configuration") -// transfer include(":transfer:transfer-00-prerequisites:connector") include(":transfer:transfer-04-event-consumer:consumer-with-listener") @@ -43,23 +41,20 @@ include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider") include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") -include("transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") -include("transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") +include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") +include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") +include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") -// advanced include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") - -//policy include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") include(":policy:policy-01-policy-enforcement:policy-functions") include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") -// modules for code samples ------------------------------------------------------------------------ include(":other:custom-runtime") -include("util:http-request-logger") +include(":util:http-request-logger") include(":system-tests") diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index f7f3e6a8..deb02448 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -39,7 +39,8 @@ dependencies { testCompileOnly(project(":transfer:transfer-04-event-consumer:listener")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) + testCompileOnly(project(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime")) testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-provider")) testCompileOnly(project(":advanced:advanced-01-open-telemetry:open-telemetry-consumer")) -} \ No newline at end of file +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java index 0e79a98b..1ff5f45e 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/FileTransferCommon.java @@ -15,10 +15,6 @@ package org.eclipse.edc.samples.common; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.edc.connector.transfer.spi.types.DataRequest; import org.eclipse.edc.junit.testfixtures.TestUtils; import org.jetbrains.annotations.NotNull; @@ -26,52 +22,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; -import java.util.Map; - -import static io.restassured.RestAssured.given; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.COMPLETED; -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; /** * Encapsulates common settings, test steps, and helper methods for transfer samples */ public class FileTransferCommon { - static final ObjectMapper MAPPER = new ObjectMapper(); - - static final String MANAGEMENT_API_URL = "http://localhost:9192/management"; - static final String CONTRACT_OFFER_FILE_PATH = "transfer/transfer-01-file-transfer/contractoffer.json"; - static final String TRANSFER_FILE_PATH = "transfer/transfer-01-file-transfer/filetransfer.json"; - static final String API_KEY_HEADER_KEY = "X-Api-Key"; - static final String API_KEY_HEADER_VALUE = "password"; - - final String sampleAssetFilePath; - final File sampleAssetFile; - final File destinationFile; - Duration timeout = Duration.ofSeconds(30); - Duration pollInterval = Duration.ofMillis(500); - - String contractNegotiationId; - String contractAgreementId; - - /** - * Creates a new {@code FileTransferSampleTestCommon} instance. - * - * @param sampleAssetFilePath Relative path starting from the root of the project to a file which will be read from for transfer. - * @param destinationFilePath Relative path starting from the root of the project where the transferred file will be written to. - */ - public FileTransferCommon(@NotNull String sampleAssetFilePath, @NotNull String destinationFilePath) { - this.sampleAssetFilePath = sampleAssetFilePath; - sampleAssetFile = getFileFromRelativePath(sampleAssetFilePath); - - destinationFile = getFileFromRelativePath(destinationFilePath); - } - /** * Resolves a {@link File} instance from a relative path. */ @@ -93,141 +49,4 @@ public static String getFileContentFromRelativePath(String relativePath) { } } - /** - * Assert that prerequisites are fulfilled before running the test. - * This assertion checks only whether the file to be copied is not existing already. - */ - void assertTestPrerequisites() { - assertThat(destinationFile).doesNotExist(); - } - - /** - * Remove files created while running the tests. - * The copied file will be deleted. - */ - void cleanTemporaryTestFiles() { - destinationFile.delete(); - } - - /** - * Assert that the file to be copied exists at the expected location. - * This method waits a duration which is defined in {@link FileTransferCommon#timeout}. - */ - void assertDestinationFileContent() { - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() - -> assertThat(destinationFile).hasSameBinaryContentAs(sampleAssetFile)); - } - - /** - * Assert that the transfer process state on the consumer is completed. - */ - void assertTransferProcessStatusConsumerSide(String transferProcessId) { - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> { - var state = getTransferProcessState(transferProcessId); - - assertThat(state).isEqualTo(COMPLETED.name()); - }); - } - - /** - * Assert that a POST request to initiate a contract negotiation is successful. - * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"} - */ - void initiateContractNegotiation() { - initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - } - - - void initiateContractNegotiation(String contractOfferFilePath) { - contractNegotiationId = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) - .when() - .post(MANAGEMENT_API_URL + "/v2/contractnegotiations") - .then() - .statusCode(HttpStatus.SC_OK) - .body("@id", not(emptyString())) - .extract() - .jsonPath() - .get("@id"); - } - - public String getTransferProcessState(String processId) { - return given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(String.format("%s/%s", MANAGEMENT_API_URL + "/v2/transferprocesses", processId)) - .then() - .statusCode(HttpStatus.SC_OK) - .extract().body().jsonPath().getString("'edc:state'"); - } - - /** - * Assert that a GET request to look up a contract agreement is successful. - * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/{UUID}"} - */ - void lookUpContractAgreementId() { - // Wait for transfer to be completed. - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> contractAgreementId = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(MANAGEMENT_API_URL + "/v2/contractnegotiations/{id}", contractNegotiationId) - .then() - .statusCode(HttpStatus.SC_OK) - .body("'edc:state'", equalTo("FINALIZED")) - .body("'edc:contractAgreementId'", not(emptyString())) - .extract().body().jsonPath().getString("'edc:contractAgreementId'") - ); - } - - /** - * Assert that a POST request to initiate transfer process is successful. - * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/filetransfer.json "http://localhost:9192/management/v2/transferprocesses"} - * - * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferCommon#TRANSFER_FILE_PATH}. - */ - String requestTransferFile(String transferFilePath) throws IOException { - var transferJsonFile = getFileFromRelativePath(transferFilePath); - var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId); - - var jsonPath = given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(requestBody) - .when() - .post(MANAGEMENT_API_URL + "/v2/transferprocesses") - .then() - .log().ifError() - .statusCode(HttpStatus.SC_OK) - .body("@id", not(emptyString())) - .extract() - .jsonPath(); - - var transferProcessId = jsonPath.getString("@id"); - - assertThat(transferProcessId).isNotEmpty(); - - return transferProcessId; - } - - String requestTransferFile() throws IOException { - return requestTransferFile(TRANSFER_FILE_PATH); - } - - /** - * Reads a transfer request file with changed value for contract agreement ID and file destination path. - * - * @param transferJsonFile A {@link File} instance pointing to a JSON transfer request file. - * @param contractAgreementId This string containing a UUID will be used as value for the contract agreement ID. - * @return An instance of {@link DataRequest} with changed values for contract agreement ID and file destination path. - * @throws IOException Thrown if there was an error accessing the file given in transferJsonFile. - */ - Map readAndUpdateDataRequestFromJsonFile(@NotNull File transferJsonFile, @NotNull String contractAgreementId) throws IOException { - var fileString = Files.readString(transferJsonFile.toPath()) - .replace("{path to destination file}", destinationFile.getAbsolutePath()) - .replace("{agreement ID}", contractAgreementId); - - return MAPPER.readValue(fileString, Map.class); - } } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java new file mode 100644 index 00000000..c2df6d60 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/KafkaSaslContainer.java @@ -0,0 +1,37 @@ +package org.eclipse.edc.samples.transfer.streaming; + +import org.jetbrains.annotations.NotNull; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Extension of the {@link KafkaContainer} that permits to set the SASL_PLAINTEXT security protocol + */ +public class KafkaSaslContainer extends KafkaContainer { + + private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.5.2"; + private final File envFile; + + public KafkaSaslContainer(@NotNull File envFile) { + super(DockerImageName.parse(KAFKA_IMAGE_NAME)); + this.withKraft(); + this.envFile = envFile; + } + + @Override + protected void configureKraft() { + super.configureKraft(); + try { + Files.readAllLines(envFile.toPath()) + .stream().map(it -> it.split("=", 2)) + .forEach(it -> this.withEnv(it[0], it[1])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java new file mode 100644 index 00000000..c12fdaee --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/streaming/Streaming03KafkaToKafkaTest.java @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial test implementation for sample + * + */ + +package org.eclipse.edc.samples.transfer.streaming; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.Json; +import okhttp3.mockwebserver.MockWebServer; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.eclipse.edc.spi.types.domain.edr.EndpointDataReference; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +import static java.time.Duration.ZERO; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.eclipse.edc.connector.transfer.spi.types.TransferProcessStates.STARTED; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; +import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; + +@Testcontainers +@EndToEndTest +public class Streaming03KafkaToKafkaTest { + + private static final String TOPIC = "topic-" + UUID.randomUUID(); + private static final String SAMPLE_NAME = "streaming-03-kafka-broker"; + private static final String RUNTIME_NAME = "streaming-03-runtime"; + private static final Path SAMPLE_FOLDER = Path.of("transfer", "streaming", SAMPLE_NAME); + private static final Path RUNTIME_PATH = SAMPLE_FOLDER.resolve(RUNTIME_NAME); + private static final Duration TIMEOUT = Duration.ofSeconds(60); + private static final Participant PROVIDER = Participant.Builder.newInstance() + .name("provider") + .id("provider") + .managementEndpoint(new Participant.Endpoint(URI.create("http://localhost:18181/management"))) + .protocolEndpoint(new Participant.Endpoint(URI.create("http://localhost:18182/protocol"))) + .controlEndpoint(new Participant.Endpoint(URI.create("http://localhost:18183/control"))) + .build(); + private static final Participant CONSUMER = Participant.Builder.newInstance() + .name("consumer") + .id("consumer") + .managementEndpoint(new Participant.Endpoint(URI.create("http://localhost:28181/management"))) + .protocolEndpoint(new Participant.Endpoint(URI.create("http://localhost:28182/protocol"))) + .controlEndpoint(new Participant.Endpoint(URI.create("http://localhost:28183/control"))) + .build(); + private static final String GROUP_ID = "group_id"; + + @Container + static KafkaContainer kafkaContainer = new KafkaSaslContainer(getFileFromRelativePath(SAMPLE_FOLDER.resolve("kafka.env").toString())) + .withLogConsumer(frame -> System.out.print(frame.getUtf8String())); + + @RegisterExtension + static EdcRuntimeExtension providerConnector = new EdcRuntimeExtension( + ":transfer:streaming:%s:%s".formatted(SAMPLE_NAME, RUNTIME_NAME), + "provider", + Map.of( + "edc.fs.config", + getFileFromRelativePath(RUNTIME_PATH.resolve("provider.properties").toString()) + .getAbsolutePath() + ) + ); + + @RegisterExtension + static EdcRuntimeExtension consumerConnector = new EdcRuntimeExtension( + ":transfer:streaming:%s:%s".formatted(SAMPLE_NAME, RUNTIME_NAME), + "consumer", + Map.of( + "edc.fs.config", + getFileFromRelativePath(RUNTIME_PATH.resolve("consumer.properties").toString()) + .getAbsolutePath() + ) + ); + + private final int httpReceiverPort = TestUtils.getFreePort(); + private final MockWebServer edrReceiverServer = new MockWebServer(); + + @BeforeEach + void setUp() throws IOException { + edrReceiverServer.start(httpReceiverPort); + } + + @Test + void streamData() throws InterruptedException, JsonProcessingException { + createAcls( + userCanAccess("User:alice", ResourceType.TOPIC, TOPIC), + userCanAccess("User:alice", ResourceType.GROUP, GROUP_ID) + ); + + PROVIDER.createAsset(getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("1-asset.json").toString()) + .replace("{{bootstrap.servers}}", kafkaContainer.getBootstrapServers()) + .replace("{{topic}}", TOPIC)); + PROVIDER.createPolicyDefinition(getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("2-policy-definition.json").toString())); + PROVIDER.createContractDefinition( + getFileContentFromRelativePath(SAMPLE_FOLDER.resolve("3-contract-definition.json").toString())); + + var destination = Json.createObjectBuilder() + .add("type", "Kafka") + .build(); + + var transferProcessPrivateProperties = Json.createObjectBuilder() + .add("receiverHttpEndpoint", "http://localhost:" + httpReceiverPort) + .build(); + var transferProcessId = CONSUMER.requestAsset(PROVIDER, "kafka-stream-asset", transferProcessPrivateProperties, destination); + + await().atMost(TIMEOUT).untilAsserted(() -> { + var state = CONSUMER.getTransferProcessState(transferProcessId); + assertThat(state).isEqualTo(STARTED.name()); + }); + + var producer = createKafkaProducer(); + var message = "message"; + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> producer + .send(new ProducerRecord<>(TOPIC, "key", message)), 0L, 100L, MICROSECONDS); + + var endpointDataReference = readEndpointDataReference(); + + try (var clientConsumer = createKafkaConsumer(endpointDataReference.getEndpoint(), endpointDataReference.getAuthKey(), endpointDataReference.getAuthCode())) { + clientConsumer.subscribe(List.of(endpointDataReference.getProperties().get(EDC_NAMESPACE + "topic").toString())); + + await().atMost(TIMEOUT).untilAsserted(() -> { + var records = clientConsumer.poll(ZERO); + assertThat(records.isEmpty()).isFalse(); + records.records(TOPIC).forEach(record -> assertThat(record.value()).isEqualTo(message)); + }); + } + + producer.close(); + } + + private EndpointDataReference readEndpointDataReference() throws InterruptedException, JsonProcessingException { + var request = edrReceiverServer.takeRequest(TIMEOUT.getSeconds(), SECONDS); + var body = request.getBody().readString(Charset.defaultCharset()); + return new ObjectMapper().readValue(body, EndpointDataReference.class); + } + + private AclBinding userCanAccess(String principal, ResourceType resourceType, String resourceName) { + var pattern = new ResourcePattern(resourceType, resourceName, PatternType.LITERAL); + var entry = new AccessControlEntry(principal, "*", AclOperation.READ, AclPermissionType.ALLOW); + return new AclBinding(pattern, entry); + } + + private void createAcls(AclBinding... bindings) { + var adminProperties = new Properties(); + adminProperties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + adminProperties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + adminProperties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin-secret\";"); + adminProperties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + try (var adminClient = AdminClient.create(adminProperties)) { + adminClient.createAcls(Arrays.stream(bindings).toList()).all().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private Producer createKafkaProducer() { + var props = new Properties(); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin-secret\";"); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + return new KafkaProducer<>(props); + } + + private static Consumer createKafkaConsumer(@NotNull String endpoint, @Nullable String authKey, @Nullable String authCode) { + var props = new Properties(); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";".formatted(authKey, authCode)); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, endpoint); + props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + return new KafkaConsumer<>(props); + } + +} diff --git a/transfer/streaming/streaming-03-kafka-broker/1-asset.json b/transfer/streaming/streaming-03-kafka-broker/1-asset.json new file mode 100644 index 00000000..a4b7d626 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/1-asset.json @@ -0,0 +1,11 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@id": "kafka-stream-asset", + "properties": { + }, + "dataAddress": { + "type": "Kafka", + "kafka.bootstrap.servers": "{{bootstrap.servers}}", + "topic": "{{topic}}" + } +} diff --git a/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json b/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json new file mode 100644 index 00000000..4919c71a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json @@ -0,0 +1,10 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "no-constraint-policy", + "policy": { + "@type": "odrl:use" + } +} diff --git a/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json b/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json new file mode 100644 index 00000000..d424ec90 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@id": "contract-definition", + "accessPolicyId": "no-constraint-policy", + "contractPolicyId": "no-constraint-policy", + "assetsSelector": [] +} diff --git a/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json b/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json new file mode 100644 index 00000000..0ec57558 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@type": "DatasetRequest", + "@id": "kafka-stream-asset", + "counterPartyAddress": "http://localhost:18182/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/transfer/streaming/streaming-03-kafka-broker/6-transfer.json b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json new file mode 100644 index 00000000..0e2c7d38 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequest", + "dataDestination": { + "type": "Kafka" + }, + "protocol": "dataspace-protocol-http", + "assetId": "stream-asset", + "contractId": "{{contract-agreement-id}}", + "connectorId": "provider", + "connectorAddress": "http://localhost:18182/protocol" +} diff --git a/transfer/streaming/streaming-03-kafka-broker/README.md b/transfer/streaming/streaming-03-kafka-broker/README.md new file mode 100644 index 00000000..046c6c85 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/README.md @@ -0,0 +1,205 @@ +# Streaming KAFKA to KAFKA + +This sample demonstrates how to set up the EDC to stream messages through Kafka. +This code is only for demonstration purposes and should not be used in production. + +## Concept + +In this sample the Data-Plane is not used, the consumer will set up a kafka client to poll the messages from the broker +using some credentials obtained from the transfer process. + +The DataFlow is managed by the [KafkaToKafkaDataFlowController](streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java), +that on flow initialization creates an `EndpointDataReference` containing the credentials that the consumer would then use +to poll the messages. + +### Run + +Build the connector runtime, which will be used both for the provider and consumer: +```shell +./gradlew :transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime:build +``` + +Run the provider and the consumer, which must be started from different terminal shells: +```shell +# provider +export EDC_FS_CONFIG=transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties +java -jar transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build/libs/connector.jar + +#consumer +export EDC_FS_CONFIG=transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties +java -jar transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build/libs/connector.jar +``` + +### Start Kafka and configure ACLs + +Kafka will be started in [KRaft mode](https://developer.confluent.io/learn/kraft/), a single broker with `SASL_PLAINTEXT` +as security protocol ([see config](kafka.env)), there will be an `admin` user, responsible for setting up ACLs and producing +messages, and `alice`, that will be used by the consumer to consume the messages. + +Run the Kafka container: +```shell +docker run --rm --name=kafka-kraft -h kafka-kraft -p 9093:9093 \ + -v "$PWD/transfer/streaming/streaming-03-kafka-broker/kafka-config":/config \ + --env-file transfer/streaming/streaming-03-kafka-broker/kafka.env \ + -e KAFKA_NODE_ID=1 \ + -e KAFKA_LISTENERS='PLAINTEXT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://0.0.0.0:9094' \ + -e KAFKA_ADVERTISED_LISTENERS='PLAINTEXT://localhost:9093,BROKER://localhost:9092' \ + -e KAFKA_PROCESS_ROLES='broker,controller' \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS='1@localhost:9094' \ + -e KAFKA_INTER_BROKER_LISTENER_NAME='BROKER' \ + -e KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER' \ + -e KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1 \ + -e CLUSTER_ID='4L6g3nShT-eMCtK--X86sw' \ + confluentinc/cp-kafka:7.5.2 +``` + +Create the topic `kafka-stream-topic` +```shell +docker exec -it kafka-kraft /bin/kafka-topics \ + --topic kafka-stream-topic --create --partitions 1 --replication-factor 1 \ + --command-config=/config/admin.properties \ + --bootstrap-server localhost:9092 +``` + +To give `alice` read permissions on the topic we need to set up ACLs: +```shell +docker exec -it kafka-kraft /bin/kafka-acls --command-config /config/admin.properties \ + --bootstrap-server localhost:9093 \ + --add --allow-principal 'User:alice' \ + --topic kafka-stream-topic \ + --group group_id \ + --operation Read +``` + +### Register Asset, Policy Definition and Contract Definition on provider + +Then put values of `kafka.bootstrap.servers`, `maxDuration` and `topic` in the [1-asset.json](1-asset.json) file replacing +their placeholders this way: +```json + "dataAddress": { + "type": "Kafka", + "kafka.bootstrap.servers": "localhost:9093", + "topic": "kafka-stream-topic" + } +``` + +Then create the Asset, the Policy Definition and the Contract Definition with these three calls: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/1-asset.json -X POST "http://localhost:18181/management/v3/assets" +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/2-policy-definition.json -X POST "http://localhost:18181/management/v2/policydefinitions" +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/3-contract-definition.json -X POST "http://localhost:18181/management/v2/contractdefinitions" +``` + +### Negotiate the contract + +The typical flow requires fetching the catalog from the consumer side and using the contract offer to negotiate a contract. +However, in this sample case, we already have the provider asset (`"kafka-stream-asset"`) so we can get the related dataset +directly with this call: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/4-get-dataset.json -X POST "http://localhost:28181/management/v2/catalog/dataset/request" -s | jq +``` + +The output will be something like: +```json +{ + "@id": "kafka-stream-asset", + "@type": "dcat:Dataset", + "odrl:hasPolicy": { + "@id": "Y29udHJhY3QtZGVmaW5pdGlvbg==:c3RyZWFtLWFzc2V0:NDlhYTUzZWEtMDUzMS00ZDkyLTg4Y2YtMGRjMTc4MmQ1NjY4", + "@type": "odrl:Set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "kafka-stream-asset" + }, + "dcat:distribution": { + "@type": "dcat:Distribution", + "dct:format": { + "@id": "HttpData" + }, + "dcat:accessService": "b24dfdbc-d17f-4d6e-9b5c-8fa71dacecfc" + }, + "edc:id": "kafka-stream-asset", + "@context": { + "dct": "https://purl.org/dc/terms/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "dcat": "https://www.w3.org/ns/dcat/", + "odrl": "http://www.w3.org/ns/odrl/2/", + "dspace": "https://w3id.org/dspace/v0.8/" + } +} +``` + +With the `odrl:hasPolicy/@id` we can now replace it in the [negotiate-contract.json](5-negotiate-contract.json) file +and negotiate the contract: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/5-negotiate-contract.json -X POST "http://localhost:28181/management/v2/contractnegotiations" -s | jq +``` + +### Start the transfer + +First we need to set up the receiver server on the consumer side that will receive the EndpointDataReference containing +the address and credentials to connect to the broker and poll the messages from the topic. For this you'll need to open +another terminal shell and run: +```shell +./gradlew util:http-request-logger:build +HTTP_SERVER_PORT=4000 java -jar util/http-request-logger/build/libs/http-request-logger.jar +``` +It will run on port 4000. + +At this point the contract agreement should already been issued, to verify that, please check the contract negotiation state with +this call, replacing `{{contract-negotiation-id}}` with the id returned by the negotiate contract call. +```shell +curl "http://localhost:28181/management/v2/contractnegotiations/{{contract-negotiation-id}}" -s | jq +``` + +If the `edc:contractAgreementId` is valued, it can be used to start the transfer, replacing it in the [6-transfer.json](6-transfer.json) +file to `{{contract-agreement-id}}` and then calling the connector with this command: +```shell +curl -H 'Content-Type: application/json' -d @transfer/streaming/streaming-03-kafka-broker/6-transfer.json -X POST "http://localhost:28181/management/v2/transferprocesses" -s | jq +``` +> Note that the destination address is `localhost:4000`, this because is where our http server is listening. + +Let's wait until the transfer state is `STARTED` state executing this call, replacing to `{{transfer-process-id}}` the id returned +by the start transfer call: +```shell +curl "http://localhost:28181/management/v2/transferprocesses/{{transfer-process-id}}" -s | jq +``` + +### Consume events +Now in the console of the `http-request-logger` we started before, the `EndpointDataReference` should have appeared: +```json +{ + "id":"8c52a781-2588-4c9b-8c70-4e5ad428eea9", + "endpoint":"localhost:9093", + "authKey":"alice", + "authCode":"alice-secret", + "properties": { + "https://w3id.org/edc/v0.0.1/ns/topic":"kafka-stream-topic" + } +} +``` + +Using these information on the consumer side we can run a `kafka-console-consumer` with the data received to consume +messages from the topic: +```shell +docker exec -it kafka-kraft /bin/kafka-console-consumer --topic kafka-stream-topic \ + --bootstrap-server localhost:9093 \ + --consumer-property group.id=group_id \ + --consumer-property security.protocol=SASL_PLAINTEXT \ + --consumer-property sasl.mechanism=PLAIN \ + --consumer-property sasl.jaas.config='org.apache.kafka.common.security.plain.PlainLoginModule required username="alice" password="alice-secret";' +``` + +### Produce events + +In another shell we can put ourselves in the provider shoes and create messages from the producer shell: +```shell +docker exec -it kafka-kraft /bin/kafka-console-producer --topic kafka-stream-topic \ + --producer.config=/config/admin.properties \ + --bootstrap-server localhost:9093 +``` + +For every message created on the provider side we will see a message on the consumer side. diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties b/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties new file mode 100644 index 00000000..65d00847 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka-config/admin.properties @@ -0,0 +1,3 @@ +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret"; diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka.env b/transfer/streaming/streaming-03-kafka-broker/kafka.env new file mode 100644 index 00000000..f5faa1c0 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka.env @@ -0,0 +1,10 @@ +KAFKA_AUTHORIZER_CLASS_NAME=org.apache.kafka.metadata.authorizer.StandardAuthorizer +KAFKA_AUTO_CREATE_TOPICS_ENABLE=true +KAFKA_LISTENER_NAME_BROKER_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret"; +KAFKA_LISTENER_NAME_CONTROLLER_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret"; +KAFKA_LISTENER_NAME_PLAINTEXT_PLAIN_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret" user_alice="alice-secret"; +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=BROKER:SASL_PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT,CONTROLLER:SASL_PLAINTEXT +KAFKA_SASL_ENABLED_MECHANISMS=PLAIN +KAFKA_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN +KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN +KAFKA_SUPER_USERS=User:admin diff --git a/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf b/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf new file mode 100644 index 00000000..12b5f8f9 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/kafka_broker_jaas.conf @@ -0,0 +1,7 @@ +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="admin" + password="admin-secret" + user_admin="admin-secret" + user_alice="alice-secret"; +}; diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts new file mode 100644 index 00000000..5c6b5c0a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/build.gradle.kts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) + implementation(libs.edc.api.observability) + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) + implementation(libs.edc.management.api) + implementation(libs.edc.dsp) + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.client) + implementation(libs.edc.transfer.data.plane) + implementation(libs.edc.transfer.pull.http.dynamic.receiver) + implementation(libs.edc.data.plane.spi) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + implementation(libs.edc.data.plane.kafka) + +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("connector.jar") +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties new file mode 100644 index 00000000..da471ae1 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/consumer.properties @@ -0,0 +1,11 @@ +web.http.port=28180 +web.http.path=/api +web.http.management.port=28181 +web.http.management.path=/management +web.http.protocol.port=28182 +web.http.protocol.path=/protocol +web.http.control.port=28183 +web.http.control.path=/control +edc.dsp.callback.address=http://localhost:28182/protocol +edc.participant.id=consumer +edc.receiver.http.dynamic.endpoint=http://localhost:4000/receiver diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties new file mode 100644 index 00000000..a06dcb9f --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/provider.properties @@ -0,0 +1,13 @@ +web.http.port=18180 +web.http.path=/api +web.http.management.port=18181 +web.http.management.path=/management +web.http.protocol.port=18182 +web.http.protocol.path=/protocol +web.http.control.port=18183 +web.http.control.path=/control +edc.dsp.callback.address=http://localhost:18182/protocol +edc.participant.id=provider +edc.ids.id=urn:connector:provider +edc.dataplane.http.sink.partition.size=1 +edc.receiver.http.dynamic.endpoint=http://localhost/not/used/in/this/sample diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java new file mode 100644 index 00000000..7657cfb2 --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaExtension.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.samples.streaming; + +import org.eclipse.edc.connector.transfer.spi.flow.DataFlowManager; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +/** + * Kafka Broker flow extension + */ +public class KafkaExtension implements ServiceExtension { + + @Override + public String name() { + return "Kafka stream extension"; + } + + @Inject + private DataFlowManager dataFlowManager; + + @Override + public void initialize(ServiceExtensionContext context) { + dataFlowManager.register(10, new KafkaToKafkaDataFlowController()); + } + +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java new file mode 100644 index 00000000..b1a11ecc --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/java/org/eclipse/edc/samples/streaming/KafkaToKafkaDataFlowController.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.samples.streaming; + +import org.eclipse.edc.connector.transfer.spi.flow.DataFlowController; +import org.eclipse.edc.connector.transfer.spi.types.DataFlowResponse; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.dataplane.kafka.schema.KafkaDataAddressSchema; +import org.eclipse.edc.policy.model.Policy; +import org.eclipse.edc.spi.response.StatusResult; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.edr.EndpointDataReference; +import org.jetbrains.annotations.NotNull; + +import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE; + +class KafkaToKafkaDataFlowController implements DataFlowController { + @Override + public boolean canHandle(TransferProcess transferProcess) { + return "Kafka".equals(transferProcess.getContentDataAddress().getType()) && "Kafka".equals(transferProcess.getDestinationType()); + } + + @Override + public @NotNull StatusResult initiateFlow(TransferProcess transferProcess, Policy policy) { + // static credentials, in a production case these should be created dynamically and an ACLs entry should be added + var username = "alice"; + var password = "alice-secret"; + + var contentDataAddress = transferProcess.getContentDataAddress(); + var kafkaDataAddress = DataAddress.Builder.newInstance() + .type(EndpointDataReference.EDR_SIMPLE_TYPE) + .property(EndpointDataReference.ID, transferProcess.getCorrelationId()) + .property(EndpointDataReference.ENDPOINT, contentDataAddress.getStringProperty("kafka.bootstrap.servers")) + .property(EndpointDataReference.AUTH_KEY, username) + .property(EndpointDataReference.AUTH_CODE, password) + .property(EDC_NAMESPACE + KafkaDataAddressSchema.TOPIC, contentDataAddress.getStringProperty(KafkaDataAddressSchema.TOPIC)) + .build(); + + return StatusResult.success(DataFlowResponse.Builder.newInstance().dataAddress(kafkaDataAddress).build()); + } + + // TODO: terminate data flow method will available in the next EDC version, it will permit to remove permissions to the user to access the topic +} diff --git a/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..849cf54a --- /dev/null +++ b/transfer/streaming/streaming-03-kafka-broker/streaming-03-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.samples.streaming.KafkaExtension diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md index d663a21b..b79ddad0 100644 --- a/transfer/transfer-00-prerequisites/README.md +++ b/transfer/transfer-00-prerequisites/README.md @@ -21,8 +21,8 @@ This sample will go through: ### 1. Build the connector -When we speak of a connector we actually mean a .jar file that is launched on a machine. -Before we can launch a connector we'll have to build the .jar file. +When we talk about a connector in the context of Eclipse Dataspace Components, we really mean a JAR file that runs on a machine. +Before we can run a connector, we need to build the JAR file. Execute this command in project root: diff --git a/transfer/transfer-01-negotiation/README.md b/transfer/transfer-01-negotiation/README.md index 885d9a6e..8ee5cd4a 100644 --- a/transfer/transfer-01-negotiation/README.md +++ b/transfer/transfer-01-negotiation/README.md @@ -73,10 +73,10 @@ curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \ ### 3. Create a contract definition on Provider -To ensure an exchange between providers and consumers, the supplier must create a contract offer for -the good, on the basis of which a contract agreement can be negotiated. The contract definition +To ensure an exchange between provider and consumer, the provider must create a contract offer for +the asset, on the basis of which a contract agreement can be negotiated. The contract definition associates policies to a selection of assets to generate the contract offers that will be put in the -catalog. In this case, the selection is empty, so every asset is attached to these policies +catalog. In this case, the selection is empty, so every asset is attached to these policies. ```bash curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ @@ -98,7 +98,7 @@ Sample output: ### 4. How to fetch catalog on consumer side -In order to offer any data, the consumer can fetch the catalog from the provider, that will contain +In order to request any data, the consumer must fetch the catalog from the provider, which contains all the contract offers available for negotiation. In our case, it will contain a single contract offer, the so-called "catalog". To get the catalog from the consumer side, you can use the following request: @@ -179,7 +179,7 @@ looks as follows: reference Of course, this is the simplest possible negotiation sequence. Later on, both connectors can also -send counter offers in addition to just confirming or declining an offer. +send counteroffers in addition to just confirming or declining an offer. ```bash curl -d @transfer/transfer-01-negotiation/resources/negotiate-contract.json \ @@ -224,7 +224,7 @@ Sample output: "edc:state": "FINALIZED", "edc:counterPartyAddress": "http://localhost:19194/protocol", "edc:callbackAddresses": [], - "edc:contractAgreementId": "MQ==:YXNzZXRJZA==:YTc4OGEwYjMtODRlZi00NWYwLTgwOWQtMGZjZTMwMGM3Y2Ey", <--------- + "edc:contractAgreementId": "0b3150be-feaf-43bc-91e1-90f050de28bd", <--------- "@context": { "dct": "https://purl.org/dc/terms/", "edc": "https://w3id.org/edc/v0.0.1/ns/", diff --git a/transfer/transfer-02-consumer-pull/README.md b/transfer/transfer-02-consumer-pull/README.md index c1e41227..4de7b639 100644 --- a/transfer/transfer-02-consumer-pull/README.md +++ b/transfer/transfer-02-consumer-pull/README.md @@ -1,6 +1,6 @@ # Implement a simple "Consumer Pull" Http transfer flow -The purpose of this sample is to show a data exchange between 2 connectors, one representing the +The purpose of this sample is to show a data exchange between two connectors, one representing the data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find more details on [Transfer data plane documentation](https://github.com/eclipse-edc/Connector/tree/main/extensions/control-plane/transfer/transfer-data-plane). @@ -25,7 +25,7 @@ order. ### 1. Start a http server -As a pre-requisite, you need to have a http server that runs on port 4000 and logs all the incoming requests, it will +As a pre-requisite, you need to have a logging webserver that runs on port 4000 and logs all the incoming requests, it will be mandatory to get the EndpointDataReference that will be used to get the data. ```bash diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md index 85952ce9..9780c9a4 100644 --- a/transfer/transfer-04-event-consumer/README.md +++ b/transfer/transfer-04-event-consumer/README.md @@ -50,13 +50,13 @@ public class TransferProcessStartedListener implements TransferProcessListener { ## Run the sample -Assuming your provider connector is still running, we can re-use the existing assets and contract definitions stored on +Assuming your provider connector and logging webserver are still running, we can re-use the existing assets and contract definitions stored on provider side. If not, set up your assets and contract definitions as described in the [Negotiation](../transfer-01-negotiation/README.md) chapter. ### 1. Build & launch the consumer with listener extension -This consumer connector is based on a different build file, hence a new jar file will be produced. +This consumer connector is based on a different build file, hence a new JAR file will be built. Make sure to terminate your current consumer connector from the previous chapters. That way we unblock the ports and can reuse the known configuration files and API calls. @@ -109,4 +109,4 @@ DEBUG 2023-10-16T09:29:46.27174 TransferProcess 762b5a0c-43fb-4b8b-8022-669043c8 If you see the `TransferProcessStartedListener received STARTED event` log message, it means that your event consumer has been configured successfully. -[Next Chapter](../transfer-05-open-telemetry/README.md) +[Next Chapter](../transfer-05-file-transfer-cloud/README.md) diff --git a/util/http-request-logger/src/main/java/org/eclipse/edc/samples/util/HttpRequestLoggerServer.java b/util/http-request-logger/src/main/java/org/eclipse/edc/samples/util/HttpRequestLoggerServer.java index 1c4b4de9..0900ebac 100644 --- a/util/http-request-logger/src/main/java/org/eclipse/edc/samples/util/HttpRequestLoggerServer.java +++ b/util/http-request-logger/src/main/java/org/eclipse/edc/samples/util/HttpRequestLoggerServer.java @@ -49,7 +49,7 @@ public void handle(HttpExchange exchange) throws IOException { System.out.println("Body:"); System.out.println(new String(exchange.getRequestBody().readAllBytes())); System.out.println("============="); - exchange.sendResponseHeaders(200, 0); + exchange.sendResponseHeaders(200, -1); } } From 78207b5eec027c4545f43def6c369d95eb52dc38 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Mon, 20 Nov 2023 16:34:48 +0100 Subject: [PATCH 28/35] update: Policy Sample update --- README.md | 8 -------- settings.gradle.kts | 5 ----- 2 files changed, 13 deletions(-) diff --git a/README.md b/README.md index b3b6f4d6..f33195c5 100644 --- a/README.md +++ b/README.md @@ -49,14 +49,6 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. -### [Policy](./policy/README.md) - -These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what -configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide -custom code for their enforcement. - -All policy samples are located in the `policy` directory. - ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/settings.gradle.kts b/settings.gradle.kts index 298e6565..7bddd3b8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,11 +48,6 @@ include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") -include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") -include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") -include(":policy:policy-01-policy-enforcement:policy-functions") -include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") - include(":other:custom-runtime") include(":util:http-request-logger") From 4dd1407249a7555c03565beb9c3548419e0d6783 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 10:13:11 +0100 Subject: [PATCH 29/35] Revert "update: Policy Sample update" This reverts commit 78207b5eec027c4545f43def6c369d95eb52dc38. --- README.md | 8 ++++++++ settings.gradle.kts | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index f33195c5..b3b6f4d6 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. +### [Policy](./policy/README.md) + +These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what +configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide +custom code for their enforcement. + +All policy samples are located in the `policy` directory. + ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/settings.gradle.kts b/settings.gradle.kts index 7bddd3b8..298e6565 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,11 @@ include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") +include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") +include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") +include(":policy:policy-01-policy-enforcement:policy-functions") +include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") + include(":other:custom-runtime") include(":util:http-request-logger") From 20b4a3c06f16ca849b4e9b4c23d195778d8fccfd Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 13:18:42 +0100 Subject: [PATCH 30/35] update: Policy Sample update --- README.md | 8 -------- settings.gradle.kts | 5 ----- 2 files changed, 13 deletions(-) diff --git a/README.md b/README.md index b3b6f4d6..f33195c5 100644 --- a/README.md +++ b/README.md @@ -49,14 +49,6 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. -### [Policy](./policy/README.md) - -These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what -configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide -custom code for their enforcement. - -All policy samples are located in the `policy` directory. - ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/settings.gradle.kts b/settings.gradle.kts index 298e6565..7bddd3b8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,11 +48,6 @@ include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") -include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") -include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") -include(":policy:policy-01-policy-enforcement:policy-functions") -include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") - include(":other:custom-runtime") include(":util:http-request-logger") From 9d55bf94b1a44d7a9c213cfa35ec4b6acf08c643 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 13:21:18 +0100 Subject: [PATCH 31/35] Revert "update: Policy Sample update" This reverts commit 20b4a3c06f16ca849b4e9b4c23d195778d8fccfd. --- README.md | 8 ++++++++ settings.gradle.kts | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index f33195c5..b3b6f4d6 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ Click the link above to learn about the transfer samples in more detail. All transfer samples are located in the `advanced` directory. +### [Policy](./policy/README.md) + +These samples deal with the topic of policies and their evaluation and enforcement. They will teach you what +configurations you need to make to enable the evaluation of specific policy rules and constraint and how to provide +custom code for their enforcement. + +All policy samples are located in the `policy` directory. + ## Contributing See [how to contribute](https://github.com/eclipse-edc/docs/blob/main/CONTRIBUTING.md). diff --git a/settings.gradle.kts b/settings.gradle.kts index 7bddd3b8..298e6565 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,11 @@ include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") +include(":policy:policy-01-policy-enforcement:policy-enforcement-provider") +include(":policy:policy-01-policy-enforcement:policy-enforcement-consumer") +include(":policy:policy-01-policy-enforcement:policy-functions") +include(":policy:policy-01-policy-enforcement:policy-enforcement-integration-tests") + include(":other:custom-runtime") include(":util:http-request-logger") From 3cf8bc7a3ef7cf9a22060861ec145a2ee3232692 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 14:03:14 +0100 Subject: [PATCH 32/35] update: Policy Sample update --- settings.gradle.kts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index acbc6b7b..ca37d687 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -44,12 +44,6 @@ include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") -include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") -include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") -include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") - -include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") -include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") include(":advanced:advanced-01-open-telemetry:open-telemetry-consumer") include(":advanced:advanced-01-open-telemetry:open-telemetry-provider") From b51cb3a0c46667789db11c3a586883bf7b4bff9b Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 14:38:59 +0100 Subject: [PATCH 33/35] update: Policy Sample update --- .../edc/sample/extension/policy/PolicyFunctionsExtension.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java index 59372d9f..f11f71d4 100644 --- a/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java +++ b/policy/policy-01-policy-enforcement/policy-functions/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -28,7 +28,6 @@ import org.eclipse.edc.policy.model.Policy; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.asset.AssetIndex; -import org.eclipse.edc.spi.asset.AssetSelectorExpression; import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; @@ -158,7 +157,7 @@ private void registerContractDefinition(ServiceExtensionContext context) { .id("1") .accessPolicyId(accessPolicy.getUid()) .contractPolicyId(contractPolicy.getUid()) - .assetsSelectorCriterion(Criterion.criterion(Asset.PROPERTY_ID,"=", "test-document")) + .assetsSelectorCriterion(Criterion.criterion(Asset.PROPERTY_ID, "=", "test-document")) .build(); contractDefinitionStore.save(contractDefinition); } From 35acdbaa2fea727cd4db8700fc91d95e7fa051b1 Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 15:55:58 +0100 Subject: [PATCH 34/35] update: Policy Sample update --- .../build.gradle.kts | 31 ----- .../BasicPolicySampleConfirmContractTest.java | 59 --------- .../BasicPolicySampleDeclineContractTest.java | 60 --------- .../BasicPolicySampleTestCommon.java | 118 ------------------ .../testFixtures/resources/contractoffer.json | 70 ----------- .../edc/samples/policy/Policy01BasicTest.java | 25 ++++ 6 files changed, 25 insertions(+), 338 deletions(-) delete mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts delete mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java delete mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java delete mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java delete mode 100644 policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts deleted file mode 100644 index ca5a0883..00000000 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/build.gradle.kts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -plugins { - `java-library` - `java-test-fixtures` -} - - -dependencies { - testImplementation(libs.edc.junit) - - testFixturesImplementation(libs.edc.junit) - testFixturesImplementation(libs.restAssured) - testFixturesImplementation(libs.awaitility) - testFixturesImplementation(libs.assertj) - testFixturesImplementation(libs.junit.jupiter.api) - - testCompileOnly(project(":policy:policy-01-policy-enforcement:policy-enforcement-provider")) -} \ No newline at end of file diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java deleted file mode 100644 index 91c4c735..00000000 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleConfirmContractTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension; - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Map; - -import static org.eclipse.edc.sample.extension.BasicPolicySampleTestCommon.getFileFromRelativePath; - - -/** - * This test class runs test for the scenario when the contract negotiation gets CONFIRMED - * It starts a connector with config.properties, containing a start and end date for which the current time is within the defined interval. - */ -@EndToEndTest -public class BasicPolicySampleConfirmContractTest { - - static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties"; - static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-policy-enforcement/contractoffer.json"; - - @RegisterExtension - static EdcRuntimeExtension connector = new EdcRuntimeExtension( - ":policy:policy-01-policy-enforcement:policy-enforcement-provider", - "provider", - Map.of( - "edc.fs.config", getFileFromRelativePath(CONNECTOR_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - final BasicPolicySampleTestCommon testUtils = new BasicPolicySampleTestCommon(); - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() { - testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - testUtils.lookUpContractAgreementConfirmed(); - } - -} \ No newline at end of file diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java deleted file mode 100644 index d54dd534..00000000 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/test/java/org/eclipse/edc/sample/extension/BasicPolicySampleDeclineContractTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension; - -import org.eclipse.edc.junit.annotations.EndToEndTest; -import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Map; - -import static org.eclipse.edc.sample.extension.BasicPolicySampleTestCommon.getFileFromRelativePath; - - -/** - * This test class runs test for the scenario when the contract negotiation gets DECLINED - * It starts a connector with config.properties, containing a start and end date for which the current time is NOT within the defined interval. - */ -@EndToEndTest -public class BasicPolicySampleDeclineContractTest { - static final String CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "policy/policy-01-policy-enforcement/policy-enforcement-provider/config.properties"; - static final String CONTRACT_OFFER_FILE_PATH = "policy/policy-01-policy-enforcement/contractoffer.json"; - - @RegisterExtension - static EdcRuntimeExtension connector = new EdcRuntimeExtension( - ":policy:policy-01-policy-enforcement:policy-enforcement-provider", - "provider", - Map.of( - // Override 'edc.samples.policy-01.constraint.date.start' & 'edc.samples.policy-01.constraint.date.end' implicitly set via property 'edc.fs.config'. - "edc.samples.policy-01.constraint.date.start", "2022-01-01T00:00:00.000+02:00", - "edc.samples.policy-01.constraint.date.end", "2022-12-31T00:00:00.000+02:00", - "edc.fs.config", getFileFromRelativePath(CONNECTOR_CONFIG_PROPERTIES_FILE_PATH).getAbsolutePath() - ) - ); - - final BasicPolicySampleTestCommon testUtils = new BasicPolicySampleTestCommon(); - - /** - * Run all sample steps in one single test. - * Note: Sample steps cannot be separated into single tests because {@link EdcRuntimeExtension} - * runs before each single test. - */ - @Test - void runSampleSteps() { - testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); - testUtils.lookUpContractAgreementDeclined(); - } -} \ No newline at end of file diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java deleted file mode 100644 index 48afa38f..00000000 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/java/org/eclipse/edc/sample/extension/BasicPolicySampleTestCommon.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension; - - -import io.restassured.RestAssured; -import io.restassured.http.ContentType; -import org.apache.http.HttpStatus; -import org.eclipse.edc.junit.testfixtures.TestUtils; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.time.Duration; - -import static org.awaitility.Awaitility.await; -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; - -/** - * Encapsulates common settings, test steps, and helper methods for the test for {@code :policy:policy-01-contract-negotiation} sample. - */ -public class BasicPolicySampleTestCommon { - - //region constant test settings - private static final String INITIATE_CONTRACT_NEGOTIATION_URI = "http://localhost:9192/management/v2/contractnegotiations"; - private static final String LOOK_UP_CONTRACT_AGREEMENT_URI = "http://localhost:9192/management/v2/contractnegotiations/{id}"; - static final String API_KEY_HEADER_KEY = "X-Api-Key"; - static final String API_KEY_HEADER_VALUE = "password"; - //endregion - - //region changeable test settings - Duration timeout = Duration.ofSeconds(30); - Duration pollInterval = Duration.ofMillis(500); - //endregion - - String contractNegotiationId; - - /** - * Creates a new {@link BasicPolicySampleTestCommon} instance. - */ - public BasicPolicySampleTestCommon() { - } - - /** - * Resolves a {@link File} instance from a relative path. - */ - @NotNull - public static File getFileFromRelativePath(String relativePath) { - return new File(TestUtils.findBuildRoot(), relativePath); - } - - /** - * Assert that a POST request to initiate a contract negotiation is successful. - * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/api/v1/management/contractnegotiations"} - */ - void initiateContractNegotiation(String contractOfferFilePath) { - contractNegotiationId = RestAssured - .given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) - .when() - .post(INITIATE_CONTRACT_NEGOTIATION_URI) - .then() - .statusCode(HttpStatus.SC_OK) - .body("id", not(emptyString())) - .extract() - .jsonPath() - .get("id"); - } - - /** - * Assert that a GET request to look up a contract agreement is successful and the {@code state} is {@code 'CONFIRMED'}. - * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/api/v1/management/contractnegotiations/{UUID}"} - */ - void lookUpContractAgreementConfirmed() { - // Wait for transfer to be completed. - await().atMost(timeout).pollInterval(pollInterval).untilAsserted(() -> RestAssured - .given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(LOOK_UP_CONTRACT_AGREEMENT_URI, contractNegotiationId) - .then() - .statusCode(HttpStatus.SC_OK) - .body("state", equalTo("CONFIRMED")) - ); - } - - /** - * Assert that a GET request to look up a contract agreement is successful and the {@code state} is {@code 'DECLINED'}. - * This method corresponds to the command in the sample: {@code curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/api/v1/management/contractnegotiations/{UUID}"} - */ - void lookUpContractAgreementDeclined() { - // Wait for transfer to be completed. - await().atMost(Duration.ofSeconds(120)).pollInterval(pollInterval).untilAsserted(() -> RestAssured - .given() - .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) - .when() - .get(LOOK_UP_CONTRACT_AGREEMENT_URI, contractNegotiationId) - .then() - .statusCode(HttpStatus.SC_OK) - .body("state", equalTo("DECLINED")) - ); - } -} \ No newline at end of file diff --git a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json b/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json deleted file mode 100644 index faa2327a..00000000 --- a/policy/policy-01-policy-enforcement/policy-enforcement-integration-tests/src/testFixtures/resources/contractoffer.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "connectorId": "provider", - "connectorAddress": "http://localhost:8282/api/v1/ids/data", - "protocol": "ids-multipart", - "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", - "assetId": "test-document", - "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [ - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2022-01-01T00:00:00.000+02:00" - }, - "operator": "GT" - }, - { - "edctype": "AtomicConstraint", - "leftExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "POLICY_EVALUATION_TIME" - }, - "rightExpression": { - "edctype": "dataspaceconnector:literalexpression", - "value": "2022-12-31T00:00:00.000+02:00" - }, - "operator": "LT" - } - ], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "ids:byteSize": null, - "asset:prop:id": "test-document", - "ids:fileName": null - } - } - } -} \ No newline at end of file diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java new file mode 100644 index 00000000..70f78489 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java @@ -0,0 +1,25 @@ +package org.eclipse.edc.samples.policy; + +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.time.Clock; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +@EndToEndTest +class Policy01BasicTest { + @RegisterExtension + static EdcRuntimeExtension consumer = new EdcRuntimeExtension( + ":policy:policy-01-policy-enforcement:policy-enforcement-consumer", + "consumer", + emptyMap() + ); + @Test + void shouldStartConsumer() { + assertThat(consumer.getContext().getService(Clock.class)).isNotNull(); + } +} From 265b53bd979d7f497b4ab1676e1eef29fe7c034f Mon Sep 17 00:00:00 2001 From: majadlymhmd Date: Tue, 21 Nov 2023 17:26:09 +0100 Subject: [PATCH 35/35] update: Policy Sample update --- .../edc/samples/policy/Policy01BasicTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java index 70f78489..50adeccd 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/policy/Policy01BasicTest.java @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + package org.eclipse.edc.samples.policy; import org.eclipse.edc.junit.annotations.EndToEndTest; @@ -12,12 +26,14 @@ @EndToEndTest class Policy01BasicTest { + @RegisterExtension static EdcRuntimeExtension consumer = new EdcRuntimeExtension( ":policy:policy-01-policy-enforcement:policy-enforcement-consumer", "consumer", emptyMap() ); + @Test void shouldStartConsumer() { assertThat(consumer.getContext().getService(Clock.class)).isNotNull();