diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e9246954..32ed35d6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -73,6 +73,7 @@ testcontainers-minio = { module = "org.testcontainers:minio", version.ref = "tes testcontainers-hashicorp-vault = { module = "org.testcontainers:vault", version.ref = "testcontainers" } azure-storage-blob = { module = "com.azure:azure-storage-blob", version = "12.28.1" } minio-io = { module = "io.minio:minio", version = "8.5.13" } - +edc-data-plane-util = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" } +edc-provision-aws-s3 = { module = "org.eclipse.edc.aws:provision-aws-s3", version.ref = "edc" } [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/policy/policy-02-provision/README.md b/policy/policy-02-provision/README.md index 22989fbe..919cf740 100644 --- a/policy/policy-02-provision/README.md +++ b/policy/policy-02-provision/README.md @@ -1,211 +1,153 @@ # Create a policy for provisioning -Now that we know how to transfer a file between two connectors and how to write policies, in this step we will see how we -can use policies to modify the supporting data transfer infrastructure. We will be regulating the file destination in a file transfer process. We will use a policy -defined in [`provision.menifest.verify`](https://eclipse-edc.github.io/docs/#/submodule/Connector/docs/developer/policy-engine?id=manifest-verification-scope-provisionmanifestverify) -scope. This scope is used during the provisioning phase to evaluate the resource definitions of a generated resource manifest. -Policy functions registered in this scope may modify resource definitions so that they comply with the policy. - -## Defining the policy for provider - -First we will create a policy definition for the provider which contains `regulateFilePathConstraint`. We can define this policy the same way we did it for the sample [`policy-01-contract-negotiation`](policy/policy-01-contract-negotiation). -The contract policy is implemented the following way: - -```java -// in PolicyFunctionsExtension.java -var desiredFilePath = context.getSetting(policyRegulatedFilePathSetting, "/tmp/provider/test-document.txt"); -var regulateFilePathConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(policyRegulateFilePath)) - .operator(Operator.EQ) - .rightExpression(new LiteralExpression(desiredFilePath)) - .build(); - -var permission = Permission.Builder.newInstance() - .action(Action.Builder.newInstance().type("USE").build()) - .constraint(regulateFilePathConstraint) - .build(); - -return PolicyDefinition.Builder.newInstance() - .id("use-regulated-path") - .policy(Policy.Builder.newInstance() - .permission(permission) - .build()) - .build(); -``` -We do not need to register any function for this policy. This is because in our current sample the `regulateFilePathConstraint` -will be applied on consumer’s `ResourceDefinition`. So we will be registering related functions in consumer’s policy functions. -We will see that in a moment. - - -## Modifying file transfer json -In previous file transfer example, while requesting a file, in [`transfer-01-file-transfer/filetransfer.json`](transfer/transfer-01-file-transfer/filetransfer.json) -we mentioned the file destination path, and set `managedResources` to `false`. But for this sample, we will set the `managedResources` to `true` and will not be -defining any file destination path. The request body for file transfer has been defined in [`filetransfer.json`](policy/policy-02-provision/filetransfer.json). - -```json -{ - ... - "dataDestination": { - "properties": { - "type": "File" - } - }, - ... - "managedResources": true, - ... -} -``` -This will allow the `ResourceManifestGenerator` to generate a `ResourceManifest`, from a `ResourceDefinition` -according to our required file type mentioned in [`filetransfer.json`](policy/policy-02-provision/filetransfer.json). +This Sample demonstrates a policy-driven approach to secure file transfers between a provider and consumer, utilizing MinIO as an S3-compatible storage solution. The setup provides a framework for managing assets with fine-grained access controls, enforced by policies. It showcases the steps required to establish assets, define and enforce policies, negotiate contracts, and transfer files according to specified policy constraints. This example will walk you through setting up MinIO, configuring connectors, and executing a complete file transfer flow with policy-regulated asset access. +In this sample, a file named `test-document` will be uploaded to the **provider-bucket** in MinIO. Once the file transfer is complete, the document will be moved to the **consumer-bucket** on the consumer’s side. -## Defining provisioner and resource definition generator -For simplicity, we are doing a local file transfer, and have implemented necessary codes required for related -resource definition generator and provisioner in the module [`policy-provision`](policy/policy-02-provision/policy-provision). -[`LocalConsumerResourceDefinitionGenerator`](policy/policy-02-provision/policy-provision/src/main/java/org/eclipse/sample/extension/provision/LocalConsumerResourceDefinitionGenerator.java) -implements `ConsumerResourceDefinitionGenerator` which generates `ResourceDefinition` for our required file type `File`. +## Prerequisites -```java -// in LocalConsumerResourceDefinitionGenerator.java +Ensure the following: +- Docker is installed: [Docker Installation](https://docs.docker.com/engine/install/). +- MinIO is configured as an S3-compatible storage emulator. -private static final String TYPE = "File"; +## Set Up MinIO -// this path will get modified during the policy evaluation to notice the change, keep the path different from the path used in policy -private static final String DESTINATION = "any path"; +### Step 1: Start MinIO -@Override -public @Nullable ResourceDefinition generate(DataRequest dataRequest, Policy policy) { - Objects.requireNonNull(dataRequest, "dataRequest must always be provided"); - Objects.requireNonNull(policy, "policy must always be provided"); +Use `docker-compose` to start MinIO: - var destination = DESTINATION; - var id = randomUUID().toString(); +```bash +docker-compose -f policy/policy-02-provision/docker-compose.yml up -d +``` - return LocalResourceDefinition.Builder.newInstance() - .id(id) - .pathName(destination) - .build(); -} -@Override -public boolean canGenerate(DataRequest dataRequest, Policy policy) { - Objects.requireNonNull(dataRequest, "dataRequest must always be provided"); - Objects.requireNonNull(policy, "policy must always be provided"); +### Step 2: Create and Configure a MinIO Bucket - return TYPE.equals(dataRequest.getDestinationType()); -} -``` +1. Go to [MinIO Console](http://localhost:9001). +2. Log in with the credentials in the `docker-compose.yml` file (line 12-13). +3. Navigate to **Buckets** and create a bucket named **provider-bucket** for storing assets. -[`LocalProvisionExtension`](policy/policy-02-provision/policy-provision/src/main/java/org/eclipse/sample/extension/provision/LocalProvisionExtension.java) -generates a [`LocalResourceProvisioner`](policy/policy-02-provision/policy-provision/src/main/java/org/eclipse/sample/extension/provision/LocalResourceProvisioner.java) -which is our required type of provisioner for local resources. +## Build and Start the Connectors +Before making the first request, build and run the provider and consumer connectors for this sample. -## Creating and registering the policy function for consumer +### Build and Run the Consumer Connector -Now, as we are willing to modify the data destination according to our policy, we have to define a policy that will -be evaluated in [`provision.menifest.verify`](https://eclipse-edc.github.io/docs/#/submodule/Connector/docs/developer/policy-engine?id=manifest-verification-scope-provisionmanifestverify) -scope. +```shell +./gradlew policy:policy-02-provision:policy-provision-consumer:build -As the data destination address is defined in consumer ResourceManifest, we have to write a policy function that will -be used by consumer connector. [`policy-provision-consumer-policy-functions`](policy/policy-02-provision/policy-provision-consumer-policy-functions) -module includes the policy functions for consumer connector. +java -Dedc.fs.config=policy/policy-02-provision/policy-provision-consumer/config.properties -jar policy/policy-02-provision/policy-provision-consumer/build/libs/consumer.jar +# for windows +java -D"edc.fs.config"=policy/policy-02-provision/policy-provision-consumer/config.properties -jar policy/policy-02-provision/policy-provision-consumer/build/libs/consumer.jar +``` +### Build and Run the Provider Connector -The [`RegulateFilePathFunction `](policy/policy-02-provision/policy-provision-consumer-policy-functions/src/main/java/org/eclipse/sample/extension/provision/consumer/policy/RegulateFilePathFunction.java) -implements the `AtomicConstraintFunction` interface, which contains a single method for evaluating a constraint. -In that method, the `operator` `EQ` and desired `pathname` in the `right value` of the constraint are used for evaluation. -In this example, we updated the `pathName` in `LocalResourceDefinition` to our desired `pathName` which was defined in our policy. +In another terminal, build and run the provider connector: +```shell +./gradlew policy:policy-02-provision:policy-provision-provider:build -Next, we have to register our function with the `PolicyEngine` and bind the desired action as well as the key used to -register our function to the desired scopes using the `RuleBindingRegistry`. This is done in the -[`ConsumerPolicyFunctionsExtension`](policy/policy-02-provision/policy-provision-consumer-policy-functions/src/main/java/org/eclipse/sample/extension/provision/consumer/policy/ConsumerPolicyFunctionsExtension.java): +java -Dedc.fs.config=policy/policy-02-provision/policy-provision-provider/config.properties -jar policy/policy-02-provision/policy-provision-provider/build/libs/provider.jar +# for windows +java -D"edc.fs.config"=policy/policy-02-provision/policy-provision-provider/config.properties -jar policy/policy-02-provision/policy-provision-provider/build/libs/provider.jar +``` -```java -private final String policyRegulateFilePath = "POLICY_REGULATE_FILE_PATH"; +## Define and Register Resources -//... -@Override -public void initialize(ServiceExtensionContext context) { - //... +### Step 1: Register the Asset on the Provider - ruleBindingRegistry.bind("USE", ALL_SCOPES); - ruleBindingRegistry.bind(policyRegulateFilePath, MANIFEST_VERIFICATION_SCOPE); - policyEngine.registerFunction(MANIFEST_VERIFICATION_SCOPE, Permission.class, policyRegulateFilePath, new RegulateFilePathFunction(monitor)); +Register an asset to make it available for transfer. - //... -} +```bash +curl -d @policy/policy-02-provision/resources/create-asset.json \ + -H 'content-type: application/json' http://localhost:8182/management/v3/assets -s | jq ``` -Here, we do not need to define any policy, as this policy function will be used by consumer connector. +### Step 2: Define an Access Policy on the Provider + +Create an access policy that will regulate file paths for this asset. + +```bash +curl -d @policy/policy-02-provision/resources/create-policy.json \ + -H 'content-type: application/json' http://localhost:8182/management/v3/policydefinitions -s | jq +``` +### Step 3: Create a Contract Definition on the Provider -## How to run the sample +Link the access policy to the asset by creating a contract definition, allowing discovery and negotiation. -Running this sample consists of the same steps done in file transfer sample. +```bash +curl -d @policy/policy-02-provision/resources/create-contract-definition.json \ + -H 'content-type: application/json' http://localhost:8182/management/v3/contractdefinitions -s | jq +``` -### Configuration +### Step 4: Retrieve provider Contract Offers -Set the desired path address in the provider [`config.properties`](policy/policy-02-provision/policy-provision-provider/config.properties). +The consumer retrieves the provider’s catalog to view available contract offers. -```properties -edc.samples.policy-02.constraint.desired.file.path = path/to/desired/location/transfer.txt +```bash +curl -X POST "http://localhost:9192/management/v3/catalog/request" \ + -H 'X-Api-Key: password' -H 'Content-Type: application/json' \ + -d @policy/policy-02-provision/resources/fetch-catalog.json -s | jq ``` -### Run the sample +Please replace the {{contract-offer-id}} placeholder in the [negotiate-contract.json](resources/negotiate-contract.json) file with the contract offer id you found in the catalog at the path dcat:dataset.odrl:hasPolicy.@id. -### 1. Build and start the connectors -First, build and run the provider and consumer connector for this sample: -Build and run the consumer connector: -```shell -./gradlew policy:policy-02-provision:policy-provision-consumer:build -java -Dedc.fs.config=policy/policy-02-provision/policy-provision-consumer/config.properties -jar policy/policy-02-provision/policy-provision-consumer/build/libs/consumer.jar -# for windows -java -D"edc.fs.config"=policy/policy-02-provision/policy-provision-consumer/config.properties -jar policy/policy-02-provision/policy-provision-consumer/build/libs/consumer.jar -``` -In another terminal, build and run the provider connector: -```shell -./gradlew policy:policy-02-provision:policy-provision-provider:build +## Contract Negotiation and Transfer -java -Dedc.fs.config=policy/policy-02-provision/policy-provision-provider/config.properties -jar policy/policy-02-provision/policy-provision-provider/build/libs/provider.jar -# for windows -java -D"edc.fs.config"=policy/policy-02-provision/policy-provision-provider/config.properties -jar policy/policy-02-provision/policy-provision-provider/build/libs/provider.jar +### Step 5: Negotiate the Contract + + +```bash +curl -d @policy/policy-02-provision/resources/negotiate-contract.json \ + -H 'X-Api-Key: password' \ + -X POST -H 'Content-Type: application/json' \ + http://localhost:9192/management/v3/contractnegotiations -s | jq ``` -### 2. Initiate a contract negotiation -Next, initiate a contract negotiation. The request body is prepared in [`contractoffer.json`](policy/policy-02-provision/contractoffer.json). -Then run: +### Step 6: Retrieve the Contract Agreement ID -```shell -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @policy/policy-02-provision/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations" +To check the status and retrieve the contract agreement ID, use the negotiation ID from the previous step. + +```bash +curl -X GET "http://localhost:9192/management/v3/contractnegotiations/{{contract-negotiation-id}}" \ + -H 'X-Api-Key: password' \ + --header 'Content-Type: application/json' -s | jq ``` -### 3. Look up the contract agreement ID +Replace `{{contract-agreement-id}}` in [filetransfer.json](resources/filetransfer.json) with the retrieved agreement ID. + +### Step 7: Initiate the File Transfer -Look up the contract agreement ID: +With the agreement in place, initiate the file transfer to the **consumer-bucket** in MinIO. ```bash -curl -X GET -H 'X-Api-Key: password' "http://localhost:9192/management/v2/contractnegotiations/" +curl -d @policy/policy-02-provision/resources/filetransfer.json \ + -H 'X-Api-Key: password' \ + -X POST -H 'Content-Type: application/json' \ + http://localhost:9192/management/v3/transferprocesses -s | jq ``` -### 4. Request the file +### Step 8: Verify Transfer Status -To request a file transfer, you need to use the [`filetransfer.json`](policy/policy-02-provision/filetransfer.json). First, locate the `contractId` field in the [`filetransfer.json`](policy/policy-02-provision/filetransfer.json) file. -Then, replace the `{{contract-agreement-id}}` placeholder with the actual contract agreement ID that you obtained from the previous step: +Use the transfer process ID to verify the file transfer status. ```bash -curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @policy/policy-02-provision/filetransfer.json "http://localhost:9192/management/v2/transferprocesses" +curl -H 'X-Api-Key: password' http://localhost:9192/management/v3/transferprocesses/ -s | jq ``` -### 5. See transferred file +### Step 9: Verify the File in MinIO -After the file transfer is completed, we can check the destination path specified in the policy/[`config.properties`](policy/policy-02-provision/policy-provision-provider/config.properties) -for the file. Here, we'll now find a file with the same content as the original file offered by the provider. We should notice that even though -`LocalConsumerResourceDefinitionGenerator` defined a different destination for the file, the path is getting modified according to the -policy. +Once the transfer is complete, check the **consumer-bucket** in MinIO for the transferred file. Access MinIO at [http://localhost:9000](http://localhost:9000) and log in with `admin` / `password`. + +## Stop docker container +Execute the following command in a terminal window to stop the docker container: +```bash +docker-compose -f policy/policy-02-provision/docker-compose.yml down +``` ---- \ No newline at end of file +--- diff --git a/policy/policy-02-provision/policy-provision-consumer/build.gradle.kts b/policy/policy-02-provision/policy-provision-consumer/build.gradle.kts index c1c056a0..fa113ebe 100644 --- a/policy/policy-02-provision/policy-provision-consumer/build.gradle.kts +++ b/policy/policy-02-provision/policy-provision-consumer/build.gradle.kts @@ -28,7 +28,7 @@ dependencies { implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.validator.data.address.http.data) - implementation(libs.edc.api.control.configuration) + implementation(libs.edc.control.api.configuration) implementation(libs.edc.edr.cache.api) implementation(libs.edc.edr.store.core) @@ -38,10 +38,13 @@ dependencies { implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.data.plane.self.registration) - implementation(libs.edc.data.plane.control.api) + implementation(libs.edc.data.plane.signaling.api) implementation(libs.edc.data.plane.public.api) implementation(libs.edc.data.plane.core) implementation(libs.edc.data.plane.http) + implementation(libs.edc.provision.aws.s3) + implementation(libs.edc.data.plane.aws.s3) + implementation(libs.edc.http) } application { diff --git a/policy/policy-02-provision/policy-provision-consumer/config.properties b/policy/policy-02-provision/policy-provision-consumer/config.properties index d0939c9c..e91d57e1 100644 --- a/policy/policy-02-provision/policy-provision-consumer/config.properties +++ b/policy/policy-02-provision/policy-provision-consumer/config.properties @@ -6,10 +6,12 @@ web.http.management.port=9192 web.http.management.path=/management web.http.protocol.port=9292 web.http.protocol.path=/protocol -edc.transfer.proxy.token.signer.privatekey.alias=private-key -edc.transfer.proxy.token.verifier.publickey.alias=public-key web.http.public.port=9195 web.http.public.path=/public web.http.control.port=9193 web.http.control.path=/control +edc.aws.access.key=S3AccessKey +edc.aws.secret.access.key=S3SecretKey +edc.aws.endpoint.override=http://localhost:9000 +edc.aws.region=us-east-1 diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/ConsumerPolicyFunctionsExtension.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/ConsumerPolicyFunctionsExtension.java index 96bfc72f..ffff4797 100644 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/ConsumerPolicyFunctionsExtension.java +++ b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/ConsumerPolicyFunctionsExtension.java @@ -20,16 +20,18 @@ import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; -import static org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator.MANIFEST_VERIFICATION_SCOPE; +//import static org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator.MANIFEST_VERIFICATION_SCOPE; import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; @Extension(value = ConsumerPolicyFunctionsExtension.NAME) public class ConsumerPolicyFunctionsExtension implements ServiceExtension { public static final String NAME = "Consumer Policy Functions Extension"; public static final String KEY = "POLICY_REGULATE_FILE_PATH"; + public static final String MANIFEST_VERIFICATION_SCOPE = "provision.manifest.verify"; @Inject private Monitor monitor; @@ -37,12 +39,17 @@ public class ConsumerPolicyFunctionsExtension implements ServiceExtension { private RuleBindingRegistry ruleBindingRegistry; @Inject private PolicyEngine policyEngine; + @Inject + private Vault vault; @Override public void initialize(ServiceExtensionContext context) { ruleBindingRegistry.bind("USE", ALL_SCOPES); ruleBindingRegistry.bind(KEY, MANIFEST_VERIFICATION_SCOPE); policyEngine.registerFunction(MANIFEST_VERIFICATION_SCOPE, Permission.class, KEY, new RegulateFilePathFunction(monitor)); + + vault.storeSecret("S3AccessKey", "admin"); + vault.storeSecret("S3SecretKey", "password"); } @Override diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/RegulateFilePathFunction.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/RegulateFilePathFunction.java index 0c932673..e221e999 100644 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/RegulateFilePathFunction.java +++ b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/policy/RegulateFilePathFunction.java @@ -15,11 +15,11 @@ package org.eclipse.edc.sample.extension.policy; import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestContext; +import org.eclipse.edc.connector.provision.aws.s3.S3BucketResourceDefinition; import org.eclipse.edc.policy.engine.spi.AtomicConstraintFunction; import org.eclipse.edc.policy.engine.spi.PolicyContext; import org.eclipse.edc.policy.model.Operator; import org.eclipse.edc.policy.model.Permission; -import org.eclipse.edc.sample.extension.provision.LocalResourceDefinition; import org.eclipse.edc.spi.monitor.Monitor; import java.util.Objects; @@ -33,13 +33,18 @@ public RegulateFilePathFunction(Monitor monitor) { @Override public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { - var desiredFilePath = (String) rightValue; + var desiredRegion = (String) rightValue; if (Objects.requireNonNull(operator) == Operator.EQ) { var manifestContext = context.getContextData(ResourceManifestContext.class); + manifestContext.getDefinitions().stream() - .filter(definition -> definition.getClass().equals(LocalResourceDefinition.class)) - .forEach(definition -> ((LocalResourceDefinition) definition).updatePathName(desiredFilePath)); + .filter(S3BucketResourceDefinition.class::isInstance) + .map(S3BucketResourceDefinition.class::cast) + .forEach(definition -> { + definition.toBuilder().regionId(desiredRegion).build(); + }); + return true; } diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalConsumerResourceDefinitionGenerator.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalConsumerResourceDefinitionGenerator.java deleted file mode 100644 index b32d2373..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalConsumerResourceDefinitionGenerator.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ConsumerResourceDefinitionGenerator; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; -import org.eclipse.edc.policy.model.Policy; -import org.jetbrains.annotations.Nullable; - -import java.util.Objects; - -import static java.util.UUID.randomUUID; - -public class LocalConsumerResourceDefinitionGenerator implements ConsumerResourceDefinitionGenerator { - - private static final String TYPE = "File"; - /** - * This will get modified during the policy evaluation to notice the change, keep the path different from the path used in policy - */ - private static final String DESTINATION = "any path"; - - @Override - public @Nullable ResourceDefinition generate(TransferProcess transferProcess, Policy policy) { - Objects.requireNonNull(transferProcess, "transferProcess must always be provided"); - Objects.requireNonNull(policy, "policy must always be provided"); - - return LocalResourceDefinition.Builder.newInstance() - .id(randomUUID().toString()) - .pathName(DESTINATION) - .build(); - } - - @Override - public boolean canGenerate(TransferProcess transferProcess, Policy policy) { - Objects.requireNonNull(transferProcess, "dataRequest must always be provided"); - Objects.requireNonNull(policy, "policy must always be provided"); - - return TYPE.equals(transferProcess.getDestinationType()); - } - -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionExtension.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionExtension.java deleted file mode 100644 index 4780387f..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionExtension.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ProvisionManager; -import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; - -@Extension(value = LocalProvisionExtension.NAME) -public class LocalProvisionExtension implements ServiceExtension { - public static final String NAME = "Local Provision Extension"; - @Setting - private static final String PROVISION_MAX_RETRY = "10"; - @Inject - private Monitor monitor; - @Inject - private ProvisionManager provisionManager; - @Inject - private ResourceManifestGenerator manifestGenerator; - - @Override - public String name() { - return NAME; - } - - @Override - public void initialize(ServiceExtensionContext context) { - var retryPolicy = (RetryPolicy) context.getService(RetryPolicy.class); - - int maxRetries = context.getSetting(PROVISION_MAX_RETRY, 10); - var provisionerConfiguration = new LocalResourceProvisionerConfiguration(maxRetries); - var localResourceProvisioner = new LocalResourceProvisioner(monitor, retryPolicy, provisionerConfiguration); - provisionManager.register(localResourceProvisioner); - - // register the generator - manifestGenerator.registerGenerator(new LocalConsumerResourceDefinitionGenerator()); - } -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionedResource.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionedResource.java deleted file mode 100644 index f0f2b8fb..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalProvisionedResource.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedDataDestinationResource; - - -@JsonDeserialize(builder = LocalProvisionedResource.Builder.class) -@JsonTypeName("dataspaceconnector:datarequest") -public class LocalProvisionedResource extends ProvisionedDataDestinationResource { - private static final String PATHNAME = "path"; - private static final String TYPE = "File"; - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder extends ProvisionedDataDestinationResource.Builder { - private Builder() { - super(new LocalProvisionedResource()); - dataAddressBuilder.type(TYPE); - } - - @JsonCreator - public static Builder newInstance() { - return new Builder(); - } - - public Builder pathName(String pathName) { - dataAddressBuilder.property(PATHNAME, pathName); - return this; - } - } -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceDefinition.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceDefinition.java deleted file mode 100644 index caa523e5..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceDefinition.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; - -import java.util.Objects; - -public class LocalResourceDefinition extends ResourceDefinition { - private String pathName; - - private LocalResourceDefinition() { - } - - public String getPathName() { - return pathName; - } - - private void setPathName(String pathName) { - this.pathName = pathName; - } - - public void updatePathName(String pathName) { - setPathName(pathName); - } - - @Override - public Builder toBuilder() { - return initializeBuilder(new Builder()) - .pathName(pathName); - } - - public static class Builder extends ResourceDefinition.Builder { - private Builder() { - super(new LocalResourceDefinition()); - } - - public static Builder newInstance() { - return new Builder(); - } - - public Builder pathName(String pathName) { - resourceDefinition.pathName = pathName; - return this; - } - - @Override - protected void verify() { - super.verify(); - Objects.requireNonNull(resourceDefinition.pathName, "pathName"); - } - } -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisioner.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisioner.java deleted file mode 100644 index fc85ee9e..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisioner.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -import dev.failsafe.RetryPolicy; -import org.eclipse.edc.connector.controlplane.transfer.spi.provision.Provisioner; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.DeprovisionedResource; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionResponse; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedResource; -import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition; -import org.eclipse.edc.policy.model.Policy; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.response.StatusResult; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.CompletableFuture; - -import static java.util.concurrent.CompletableFuture.completedFuture; - - -public class LocalResourceProvisioner implements Provisioner { - private final Monitor monitor; - private final RetryPolicy retryPolicy; - private final LocalResourceProvisionerConfiguration configuration; - - public LocalResourceProvisioner(Monitor monitor, RetryPolicy retryPolicy, LocalResourceProvisionerConfiguration configuration) { - this.monitor = monitor; - this.configuration = configuration; - this.retryPolicy = RetryPolicy.builder(retryPolicy.getConfig()) - .withMaxRetries(configuration.maxRetries()) - .build(); - } - - @Override - public boolean canProvision(ResourceDefinition resourceDefinition) { - return resourceDefinition instanceof LocalResourceDefinition; - } - - @Override - public boolean canDeprovision(ProvisionedResource resourceDefinition) { - return resourceDefinition instanceof LocalProvisionedResource; - } - - @Override - public CompletableFuture> provision(LocalResourceDefinition resourceDefinition, Policy policy) { - createDestinationFile(resourceDefinition.getPathName()); - StatusResult provisionResponseStatusResult = provisionSucceeded(resourceDefinition); - return completedFuture(provisionResponseStatusResult); - } - - @Override - public CompletableFuture> deprovision(LocalProvisionedResource provisionedResource, Policy policy) { - return null; - } - - - private void createDestinationFile(String pathName) { - var file = new File(pathName.replaceAll("\\.", ".").replaceAll("/", "/")); - if (!file.exists()) { - try { - if (!file.createNewFile()) { - monitor.debug(String.format("File could not be created at path %s", pathName)); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - private StatusResult provisionSucceeded(LocalResourceDefinition resourceDefinition) { - var resource = LocalProvisionedResource.Builder.newInstance() - .id(resourceDefinition.getPathName()) - .resourceDefinitionId(resourceDefinition.getId()) - .hasToken(true) - .pathName(resourceDefinition.getPathName()) - .transferProcessId(resourceDefinition.getTransferProcessId()) - .resourceName(resourceDefinition.getPathName()) - .build(); - - monitor.debug("LocalResourceProvisioner: Resource request submitted: " + resourceDefinition.getPathName()); - - var response = ProvisionResponse.Builder.newInstance().resource(resource).build(); - return StatusResult.success(response); - } -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisionerConfiguration.java b/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisionerConfiguration.java deleted file mode 100644 index 89731858..00000000 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/java/org/eclipse/edc/sample/extension/provision/LocalResourceProvisionerConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (c) 2023 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.provision; - -public record LocalResourceProvisionerConfiguration(int maxRetries) { - -} diff --git a/policy/policy-02-provision/policy-provision-consumer/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-02-provision/policy-provision-consumer/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 0662e409..6b7a7971 100644 --- a/policy/policy-02-provision/policy-provision-consumer/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/policy/policy-02-provision/policy-provision-consumer/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -12,5 +12,5 @@ # # -org.eclipse.edc.sample.extension.provision.LocalProvisionExtension +#org.eclipse.edc.sample.extension.provision.LocalProvisionExtension org.eclipse.edc.sample.extension.policy.ConsumerPolicyFunctionsExtension \ No newline at end of file diff --git a/policy/policy-02-provision/policy-provision-provider/build.gradle.kts b/policy/policy-02-provision/policy-provision-provider/build.gradle.kts index 50c8c041..09946b18 100644 --- a/policy/policy-02-provision/policy-provision-provider/build.gradle.kts +++ b/policy/policy-02-provision/policy-provision-provider/build.gradle.kts @@ -29,7 +29,7 @@ dependencies { implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.validator.data.address.http.data) - implementation(libs.edc.api.control.configuration) + implementation(libs.edc.control.api.configuration) implementation(libs.edc.edr.cache.api) implementation(libs.edc.edr.store.core) @@ -40,12 +40,12 @@ dependencies { implementation(libs.edc.data.plane.util) implementation(libs.edc.data.plane.self.registration) - implementation(libs.edc.data.plane.control.api) + implementation(libs.edc.data.plane.signaling.api) implementation(libs.edc.data.plane.public.api) implementation(libs.edc.data.plane.core) implementation(libs.edc.data.plane.http) - - //implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) + implementation(libs.edc.data.plane.aws.s3) + implementation(libs.edc.http) } application { diff --git a/policy/policy-02-provision/policy-provision-provider/config.properties b/policy/policy-02-provision/policy-provision-provider/config.properties index 67341053..fcfd38af 100644 --- a/policy/policy-02-provision/policy-provision-provider/config.properties +++ b/policy/policy-02-provision/policy-provision-provider/config.properties @@ -1,17 +1,18 @@ edc.participant.id=provider edc.dsp.callback.address=http://localhost:8282/protocol + web.http.port=8181 web.http.path=/api web.http.management.port=8182 web.http.management.path=/management web.http.protocol.port=8282 web.http.protocol.path=/protocol -edc.transfer.proxy.token.signer.privatekey.alias=private-key -edc.transfer.proxy.token.verifier.publickey.alias=public-key web.http.public.port=8185 web.http.public.path=/public web.http.control.port=8183 web.http.control.path=/control -edc.samples.policy-02.asset.path=path/to/file -edc.samples.policy-02.constraint.desired.file.path=path/to/desired/location/transfer.txt -edc.dataplane.api.public.baseurl=http://localhost:8185/public \ No newline at end of file + +edc.aws.access.key=admin +edc.aws.secret.access.key=password +edc.aws.endpoint.override=http://localhost:9000 +edc.aws.region=us-east-1 diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java index 6bbb8be2..7de62321 100644 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java +++ b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/policy/PolicyFunctionsExtension.java @@ -14,110 +14,38 @@ package org.eclipse.edc.sample.extension.policy; -import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; +import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.contract.spi.offer.store.ContractDefinitionStore; -import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractDefinition; -import org.eclipse.edc.connector.controlplane.policy.spi.PolicyDefinition; import org.eclipse.edc.connector.controlplane.policy.spi.store.PolicyDefinitionStore; -import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; -import org.eclipse.edc.policy.model.Action; -import org.eclipse.edc.policy.model.AtomicConstraint; -import org.eclipse.edc.policy.model.LiteralExpression; -import org.eclipse.edc.policy.model.Operator; -import org.eclipse.edc.policy.model.Permission; -import org.eclipse.edc.policy.model.Policy; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.query.Criterion; +import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; -import java.util.List; - -import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; - -@Extension(value = PolicyFunctionsExtension.NAME) +@Extension("MinIO S3-Based Transfer") public class PolicyFunctionsExtension implements ServiceExtension { - private static final String FILE_PATH = "edc.samples.policy-02.constraint.desired.file.path"; - private static final String KEY = "POLICY_REGULATE_FILE_PATH"; - public static final String NAME = "Policy Functions Extension"; - public static final String POLICY_TYPE = "USE"; - public static final String RIGHT_OPERAND = "test-document"; - public static final String DEFAULT_FILE_PATH = "/tmp/desired/path/transfer.txt"; @Inject - private RuleBindingRegistry ruleBindingRegistry; + private AssetIndex assetIndex; @Inject - private PolicyDefinitionStore policyStore; + private PolicyDefinitionStore policyDefinitionStore; @Inject private ContractDefinitionStore contractDefinitionStore; + @Inject + private Monitor monitor; @Override public String name() { - return NAME; + return "MinIO-Based Transfer"; } @Override public void initialize(ServiceExtensionContext context) { - ruleBindingRegistry.bind(POLICY_TYPE, ALL_SCOPES); - - registerContractDefinition(context); - } - - private PolicyDefinition createAccessPolicy() { - var usePermission = Permission.Builder.newInstance() - .action(Action.Builder.newInstance().type("USE").build()) - .build(); - - return PolicyDefinition.Builder.newInstance() - .id("use") - .policy(Policy.Builder.newInstance() - .permission(usePermission) - .build()) - .build(); - } + monitor.info("PolicyFunctionsExtension initialized."); - private PolicyDefinition createContractPolicy(ServiceExtensionContext context) { - var desiredFilePath = context.getSetting(FILE_PATH, DEFAULT_FILE_PATH); - var regulateFilePathConstraint = AtomicConstraint.Builder.newInstance() - .leftExpression(new LiteralExpression(KEY)) - .operator(Operator.EQ) - .rightExpression(new LiteralExpression(desiredFilePath)) - .build(); - - - var permission = Permission.Builder.newInstance() - .action(Action.Builder.newInstance().type(POLICY_TYPE).build()) - .constraint(regulateFilePathConstraint) - .build(); - - - return PolicyDefinition.Builder.newInstance() - .id("use-regulated-path") - .policy(Policy.Builder.newInstance() - .permission(permission) - .build()) - .build(); } - private void registerContractDefinition(ServiceExtensionContext context) { - var accessPolicy = createAccessPolicy(); - policyStore.create(accessPolicy); - - var contractPolicy = createContractPolicy(context); - policyStore.create(contractPolicy); +} - var contractDefinition = ContractDefinition.Builder.newInstance() - .id("1") - .accessPolicyId(accessPolicy.getId()) - .contractPolicyId(contractPolicy.getId()) - .assetsSelector(List.of(Criterion.Builder.newInstance() - .operandLeft(Asset.PROPERTY_ID) - .operator("=") // TODO changed to EQ? - .operandRight(RIGHT_OPERAND) - .build())) - .build(); - contractDefinitionStore.save(contractDefinition); - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSink.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSink.java deleted file mode 100644 index 89e0418a..00000000 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSink.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink; - -import java.io.File; -import java.io.FileOutputStream; -import java.util.List; -import java.util.Objects; - -import static java.lang.String.format; -import static org.eclipse.edc.connector.dataplane.spi.pipeline.StreamFailure.Reason.GENERAL_ERROR; - -class FileTransferDataSink extends ParallelSink { - private File file; - - @Override - protected StreamResult transferParts(List parts) { - for (DataSource.Part part : parts) { - var fileName = part.name(); - try (var input = part.openStream()) { - try (var output = new FileOutputStream(file)) { - try { - input.transferTo(output); - } catch (Exception e) { - return getTransferResult(e, "Error transferring file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error creating file %s", fileName); - } - } catch (Exception e) { - return getTransferResult(e, "Error reading file %s", fileName); - } - } - return StreamResult.success(null); - } - - private StreamResult getTransferResult(Exception e, String logMessage, Object... args) { - var message = format(logMessage, args); - monitor.severe(message, e); - return StreamResult.failure(new StreamFailure(List.of(message), GENERAL_ERROR)); - } - - public static class Builder extends ParallelSink.Builder { - - public static Builder newInstance() { - return new Builder(); - } - - public Builder file(File file) { - sink.file = file; - return this; - } - - @Override - protected void validate() { - Objects.requireNonNull(sink.file, "file"); - } - - private Builder() { - super(new FileTransferDataSink()); - } - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSinkFactory.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSinkFactory.java deleted file mode 100644 index 98e77124..00000000 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSinkFactory.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSink; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSinkFactory; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.util.concurrent.ExecutorService; - -public class FileTransferDataSinkFactory implements DataSinkFactory { - private final Monitor monitor; - private final ExecutorService executorService; - private final int partitionSize; - - public FileTransferDataSinkFactory(Monitor monitor, ExecutorService executorService, - int partitionSize) { - this.monitor = monitor; - this.executorService = executorService; - this.partitionSize = partitionSize; - } - - @Override - public String supportedType() { - return "File"; - } - - @Override - public boolean canHandle(DataFlowStartMessage request) { - return "File".equalsIgnoreCase(request.getDestinationDataAddress().getType()); - } - - @Override - public DataSink createSink(DataFlowStartMessage request) { - var destination = request.getDestinationDataAddress(); - - // verify destination path - var path = destination.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - var destinationFile = new File(path.replaceAll("\\.", ".").replaceAll("/", "/")); - - return FileTransferDataSink.Builder.newInstance() - .file(destinationFile) - .requestId(request.getId()) - .partitionSize(partitionSize) - .executorService(executorService) - .monitor(monitor) - .build(); - } - - @Override - public @NotNull Result validateRequest(DataFlowStartMessage request) { - return Result.success(); - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSource.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSource.java deleted file mode 100644 index 85ef937e..00000000 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSource.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult; -import org.eclipse.edc.spi.EdcException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.util.stream.Stream; - -class FileTransferDataSource implements DataSource { - - private final File file; - - FileTransferDataSource(File file) { - this.file = file; - } - - @Override - public StreamResult> openPartStream() { - var part = new Part() { - @Override - public String name() { - return file.getName(); - } - - @Override - public InputStream openStream() { - try { - return new FileInputStream(file); - } catch (FileNotFoundException e) { - throw new EdcException(e); - } - } - }; - return StreamResult.success(Stream.of(part)); - } - - @Override - public void close() throws Exception { - - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSourceFactory.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSourceFactory.java deleted file mode 100644 index 7a4fe144..00000000 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferDataSourceFactory.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSourceFactory; -import org.eclipse.edc.spi.result.Result; -import org.eclipse.edc.spi.types.domain.transfer.DataFlowStartMessage; -import org.jetbrains.annotations.NotNull; - -import java.io.File; - -public class FileTransferDataSourceFactory implements DataSourceFactory { - @Override - public String supportedType() { - return "File"; - } - - @Override - public boolean canHandle(DataFlowStartMessage dataRequest) { - return "File".equalsIgnoreCase(dataRequest.getSourceDataAddress().getType()); - } - - @Override - public DataSource createSource(DataFlowStartMessage request) { - var source = getFile(request); - return new FileTransferDataSource(source); - } - - @Override - public @NotNull Result validateRequest(DataFlowStartMessage request) { - var source = getFile(request); - if (!source.exists()) { - return Result.failure("Source file " + source.getName() + " does not exist at " + source.getAbsolutePath()); - } - - return Result.success(); - } - - @NotNull - private File getFile(DataFlowStartMessage request) { - var dataAddress = request.getSourceDataAddress(); - // verify source path - var sourceFileName = dataAddress.getStringProperty("filename"); - var path = dataAddress.getStringProperty("path"); - // As this is a controlled test input below is to avoid path-injection warning by CodeQL - sourceFileName = sourceFileName.replaceAll("\\.", ".").replaceAll("/", "/"); - path = path.replaceAll("\\.", ".").replaceAll("/", "/"); - return new File(path, sourceFileName); - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferExtension.java b/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferExtension.java deleted file mode 100644 index 60916992..00000000 --- a/policy/policy-02-provision/policy-provision-provider/src/main/java/org/eclipse/edc/sample/extension/transfer/FileTransferExtension.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2021 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -package org.eclipse.edc.sample.extension.transfer; - -import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; -import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; -import org.eclipse.edc.connector.controlplane.transfer.spi.flow.DataFlowManager; -import org.eclipse.edc.connector.dataplane.spi.pipeline.DataTransferExecutorServiceContainer; -import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; -import org.eclipse.edc.runtime.metamodel.annotation.Extension; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.domain.DataAddress; - -import java.nio.file.Path; - -@Extension(FileTransferExtension.NAME) -public class FileTransferExtension implements ServiceExtension { - public static final String NAME = "File Transfer Extension"; - private static final String EDC_ASSET_PATH = "edc.samples.policy-02.asset.path"; - private static final String DEFAULT_PATH = "/tmp/provider/test-document.txt"; - - @Inject - private AssetIndex assetIndex; - @Inject - private PipelineService pipelineService; - @Inject - private DataTransferExecutorServiceContainer executorContainer; - @Inject - private Monitor monitor; - @Inject - private DataFlowManager dataFlowManager; - - @Override - public void initialize(ServiceExtensionContext context) { - pipelineService.registerFactory(new FileTransferDataSourceFactory()); - - var sinkFactory = new FileTransferDataSinkFactory(monitor, executorContainer.getExecutorService(), 5); - pipelineService.registerFactory(sinkFactory); - - registerDataEntries(context); - } - - private void registerDataEntries(ServiceExtensionContext context) { - var assetPathSetting = context.getSetting(EDC_ASSET_PATH, DEFAULT_PATH); - var assetPath = Path.of(assetPathSetting); - var filename = assetPath.getFileName().toString(); - var path = assetPath.getParent().toString(); - - var dataAddress = DataAddress.Builder.newInstance() - .property("type", "File") - .property("filename", filename) - .property("path", path) - .build(); - - - var asset = Asset.Builder.newInstance() - .id("test-document") - .dataAddress(dataAddress) - .build(); - - assetIndex.create(asset); - } -} diff --git a/policy/policy-02-provision/policy-provision-provider/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/policy/policy-02-provision/policy-provision-provider/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension index 9b4a0fd1..ac291479 100644 --- a/policy/policy-02-provision/policy-provision-provider/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ b/policy/policy-02-provision/policy-provision-provider/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -13,4 +13,4 @@ # org.eclipse.edc.sample.extension.policy.PolicyFunctionsExtension -org.eclipse.edc.sample.extension.transfer.FileTransferExtension \ No newline at end of file +#org.eclipse.edc.sample.extension.transfer.FileTransferExtension \ No newline at end of file diff --git a/policy/policy-02-provision/resources/bucket-policy.json b/policy/policy-02-provision/resources/bucket-policy.json new file mode 100644 index 00000000..975e66ce --- /dev/null +++ b/policy/policy-02-provision/resources/bucket-policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + } + ] +} diff --git a/policy/policy-02-provision/resources/create-asset.json b/policy/policy-02-provision/resources/create-asset.json new file mode 100644 index 00000000..0a48ebda --- /dev/null +++ b/policy/policy-02-provision/resources/create-asset.json @@ -0,0 +1,15 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "test-document", + "properties": { + "name": "Test Document", + "contenttype": "application/json" + }, + "dataAddress": { + "type": "HttpData", + "baseUrl": "https://jsonplaceholder.typicode.com/users", + "proxyPath": "true" + } +} diff --git a/policy/policy-02-provision/resources/create-contract-definition.json b/policy/policy-02-provision/resources/create-contract-definition.json new file mode 100644 index 00000000..659677e3 --- /dev/null +++ b/policy/policy-02-provision/resources/create-contract-definition.json @@ -0,0 +1,16 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "ContractDefinition", + "@id": "1", + "accessPolicyId": "accessPolicy", + "contractPolicyId": "accessPolicy", + "assetsSelector": [ + { + "operandLeft": "https://w3id.org/edc/v0.0.1/ns/id", + "operator": "=", + "operandRight": "test-document" + } + ] +} diff --git a/policy/policy-02-provision/resources/create-policy.json b/policy/policy-02-provision/resources/create-policy.json new file mode 100644 index 00000000..8457c2a1 --- /dev/null +++ b/policy/policy-02-provision/resources/create-policy.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "accessPolicy", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@type": "Set", + "permission": [ + { + "action": { + "@id": "USE" + } + } + ], + "prohibition": [], + "obligation": [] + } +} diff --git a/policy/policy-02-provision/resources/docker-compose.yml b/policy/policy-02-provision/resources/docker-compose.yml new file mode 100644 index 00000000..b613bb2f --- /dev/null +++ b/policy/policy-02-provision/resources/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.9" + +services: + minio: + image: docker.io/bitnami/minio:latest + ports: + - '9000:9000' + - '9001:9001' + volumes: + - 'minio_data:/data' + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + + mc: + image: minio/mc + depends_on: + - minio + volumes: + - ./provider-bucket:/provider-bucket + entrypoint: > + /bin/sh -c /provider-bucket/init.sh + +volumes: + minio_data: + driver: local diff --git a/policy/policy-02-provision/resources/fetch-catalog.json b/policy/policy-02-provision/resources/fetch-catalog.json new file mode 100644 index 00000000..07c326cd --- /dev/null +++ b/policy/policy-02-provision/resources/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "counterPartyAddress": "http://localhost:8282/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/policy/policy-02-provision/resources/filetransfer.json b/policy/policy-02-provision/resources/filetransfer.json new file mode 100644 index 00000000..a3888874 --- /dev/null +++ b/policy/policy-02-provision/resources/filetransfer.json @@ -0,0 +1,21 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:8282/protocol", + "contractId": "{{contract-agreement-id}}", + "assetId": "test-document", + "protocol": "dataspace-protocol-http", + "transferType": "AmazonS3-PUSH", + "dataDestination": { + "type": "AmazonS3", + "region": "us-east-1", + "bucketName": "consumer-bucket", + "keyName": "test-document.txt", + "endpointOverride": "http://localhost:9000", + "accessKeyId": "admin", + "secretAccessKey": "password" + } +} diff --git a/policy/policy-02-provision/resources/get-dataset.json b/policy/policy-02-provision/resources/get-dataset.json new file mode 100644 index 00000000..46149e24 --- /dev/null +++ b/policy/policy-02-provision/resources/get-dataset.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/"}, + "@type": "DatasetRequest", + "@id": "1", + "counterPartyAddress": "http://localhost:8282/protocol", + "protocol": "dataspace-protocol-http" +} \ No newline at end of file diff --git a/policy/policy-02-provision/resources/init.sh b/policy/policy-02-provision/resources/init.sh new file mode 100644 index 00000000..b1f1902c --- /dev/null +++ b/policy/policy-02-provision/resources/init.sh @@ -0,0 +1,16 @@ +sleep 10 +/usr/bin/mc alias set myminio http://minio:9000 admin password + +/usr/bin/mc admin user add myminio providerAccessKeyId providerSecretAccessKey +/usr/bin/mc admin user add myminio consumerAccessKeyId consumerSecretAccessKey + +/usr/bin/mc mb myminio/provider +/usr/bin/mc mb myminio/consumer + +/usr/bin/mc admin policy create myminio allowall /provider-bucket/bucket-policy.json +/usr/bin/mc admin policy attach myminio allowall --user providerAccessKeyId +/usr/bin/mc admin policy attach myminio allowall --user consumerAccessKeyId + +/usr/bin/mc cp /provider-bucket/test-document.txt myminio/provider + +exit 0 diff --git a/policy/policy-02-provision/resources/negotiate-contract.json b/policy/policy-02-provision/resources/negotiate-contract.json new file mode 100644 index 00000000..166d7364 --- /dev/null +++ b/policy/policy-02-provision/resources/negotiate-contract.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "ContractRequest", + "counterPartyAddress": "http://localhost:8282/protocol", + "protocol": "dataspace-protocol-http", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@id": "{{contract-offer-id}}", + "@type": "odrl:Offer", + "assigner": "provider", + "target": "test-document", + "odrl:permission": { + "odrl:action": { + "odrl:type": "USE" + } + } + } +} diff --git a/policy/policy-02-provision/resources/test-document.txt b/policy/policy-02-provision/resources/test-document.txt new file mode 100644 index 00000000..b18c75c8 --- /dev/null +++ b/policy/policy-02-provision/resources/test-document.txt @@ -0,0 +1 @@ +Test test \ No newline at end of file