feat: add support for minio and azurite for cloud sample

system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer05fileTransferCloudTest.java

@@ -14,15 +14,11 @@
 
 package org.eclipse.edc.samples.transfer;
 
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobContainerClientBuilder;
 import com.azure.storage.common.StorageSharedKeyCredential;
 import io.minio.ListObjectsArgs;
 import io.minio.MakeBucketArgs;
 import io.minio.MinioClient;
-import io.minio.Result;
-import io.minio.messages.Item;
 import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcessStates;
 import org.eclipse.edc.junit.annotations.EndToEndTest;
 import org.eclipse.edc.junit.extensions.EmbeddedRuntime;
@@ -43,12 +39,13 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.fail;
 import static org.eclipse.edc.samples.common.FileTransferCloudCommon.runNegotiation;
 import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath;
 import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath;
 import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus;
 import static org.eclipse.edc.samples.util.TransferUtil.startTransfer;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @Testcontainers
 @EndToEndTest
@@ -104,9 +101,9 @@ static void tearDown() {
             .withExposedPorts(VAULT_PORT)
             .withVaultToken(VAULT_TOKEN)
             .withInitCommand(
-                "vault kv put secret/accessKeyId content=" + MINIO_ACCOUNT_NAME,
-                "vault kv put secret/secretAccessKey content="  + MINIO_ACCOUNT_KEY,
-                "vault kv put secret/provider-key content=" + AZURITE_ACCOUNT_KEY
+                    "kv put secret/accessKeyId content=" + MINIO_ACCOUNT_NAME,
+                    "kv put secret/secretAccessKey content=" + MINIO_ACCOUNT_KEY,
+                    "kv put secret/provider-key content=" + AZURITE_ACCOUNT_KEY
             )
             .withLogConsumer((OutputFrame outputFrame) -> System.out.print(outputFrame.getUtf8String()));
 
@@ -151,9 +148,7 @@ EDC_FS_CONFIG, getFileFromRelativePath(CLOUD_CONSUMER_CONFIG_PROPERTIES_FILE_PAT
                     Map.entry("web.http.public.path", "/public"),
                     Map.entry("web.http.control.port", "19192"),
                     Map.entry("web.http.control.path", "/control"),
-                    Map.entry("edc.dataplane.api.public.baseurl", "http://localhost:19291/public"),
-                    Map.entry("edc.converter.usefilesystem", "false"),
-                    Map.entry("edc.vault.hashicorp.url", "http://127.0.0.1:8200"),
+                    Map.entry("edc.vault.hashicorp.url", "http://127.0.0.1:" + getVaultPort()),
                     Map.entry("edc.vault.hashicorp.token", "<root-token>"),
                     Map.entry("edc.vault.hashicorp.api.secret.path", "/v1/secret"),
                     Map.entry("edc.vault.hashicorp.health.check.enabled", "false"),
@@ -167,7 +162,7 @@ EDC_FS_CONFIG, getFileFromRelativePath(CLOUD_CONSUMER_CONFIG_PROPERTIES_FILE_PAT
     @Test
     void pushFile() throws Exception {
 
-        MinioClient minioClient =
+        var minioClient =
                 MinioClient.builder()
                 .endpoint("http://" + minioContainer.getHost() + ":" + minioContainer.getMappedPort(MINIO_PORT))
                 .credentials(MINIO_ACCOUNT_NAME, MINIO_ACCOUNT_KEY)
@@ -184,41 +179,43 @@ void pushFile() throws Exception {
 
         checkTransferStatus(transferProcessId, TransferProcessStates.COMPLETED);
 
-
-        Iterable<Result<Item>> results = minioClient.listObjects(
+        var objects = minioClient.listObjects(
                 ListObjectsArgs.builder().bucket(MINIO_BUCKET_NAME).build());
 
-        Result<Item> result = results.iterator().next();
-        Item item = result.get();
-
-        boolean objectFound = item.objectName().equals(FILE_NAME);
-
-        assertTrue(objectFound, "test-document.txt could not be found in minio");
-
+        assertThat(objects)
+                .isNotEmpty().first()
+                .extracting(result -> {
+                    try {
+                        return result.get();
+                    } catch (Exception e) {
+                        return fail();
+                    }
+                })
+                .satisfies(item -> assertThat(item.objectName()).isEqualTo(FILE_NAME));
     }
 
     private static void configureAzurite() {
-        String blobServiceUrl = String.format("http://%s:%d/%s", 
+
+        var blobServiceUrl = String.format("http://%s:%d/%s",
                 azuriteContainer.getHost(),
                 azuriteContainer.getMappedPort(AZURITE_PORT),
                 AZURITE_ACCOUNT_NAME);
 
-        StorageSharedKeyCredential credential = new StorageSharedKeyCredential(AZURITE_ACCOUNT_NAME, AZURITE_ACCOUNT_KEY);
+        var credential = new StorageSharedKeyCredential(AZURITE_ACCOUNT_NAME, AZURITE_ACCOUNT_KEY);
 
-        BlobContainerClient blobContainerClient = new BlobContainerClientBuilder()
+        var blobContainerClient = new BlobContainerClientBuilder()
                 .endpoint(blobServiceUrl)
                 .credential(credential)
                 .containerName(AZURITE_CONTAINER_NAME)
                 .buildClient();
 
         blobContainerClient.create();
 
-        BlobClient blobClient = blobContainerClient.getBlobClient(FILE_NAME);
-        String blobContent = "Test";
+        var blobClient = blobContainerClient.getBlobClient(FILE_NAME);
+        var blobContent = "Test";
 
         blobClient.upload(new ByteArrayInputStream(blobContent.getBytes(StandardCharsets.UTF_8)), blobContent.length());
 
-        assertTrue(blobClient.exists(), "Blob could not be created");
     }
 
     private static int getAzuritePort() {
@@ -231,4 +228,13 @@ private static int getAzuritePort() {
         return azuriteContainer.getMappedPort(AZURITE_PORT);
     }
 
+    private static int getVaultPort() {
+
+        if (!vaultContainer.isRunning()) {
+            vaultContainer.start();
+        }
+
+        return vaultContainer.getMappedPort(VAULT_PORT);
+    }
+
 }

transfer/transfer-05-file-transfer-cloud/README.md

@@ -1,35 +1,40 @@
 # Improve the file transfer
 
-So far we have performed a file transfer on a local machine using the Eclipse Dataspace Connector. While that is already great progress, it probably won't be much use in a real-world production application.
+So far, we have performed a file transfer on a local machine using the Eclipse Dataspace Connector. While that is already great progress, it probably won't be much use in a real-world production application.
 
-This chapter improves on this by moving the file transfer "to the cloud". What we mean by that is that instead of reading and writing the file from/to the disk, we will now:
+This chapter improves on this by shifting the file transfer between cloud storage emulators. We will now:
 
-- read the source from an Azure Storage,
-- put the destination file into an AWS S3 Bucket.
+- read the source from an Azurite instance,
+- put the destination file into a MinIO instance.
 
-But instead of real clouds we will use docker containers for the transfer. For that, you have to install docker on your machine.
+## Prerequisites
+
+The following steps assume that you have Docker and the Azure CLI installed. If this is not the case, you can use the following links to access the installation instructions for both.
+
+- Docker Desktop: https://docs.docker.com/engine/install/
+- Azure CLI: https://learn.microsoft.com/en-us/cli/azure/install-azure-cli
 
 ## Start the docker-compose file
 
 ```bash
-docker-compose -f transfer/transfer-05-file-transfer-cloud/resources/docker-compose.yaml up -d
+docker compose -f transfer/transfer-05-file-transfer-cloud/resources/docker-compose.yaml up -d
 ```
 
 Please check in the logs that minio, azurite and hashicorp-vault have started correctly.  
 
 ## Create bucket in minio
 
-Go to http://localhost:9001 and login with the credentials which you can find in the [docker-compose](resources/docker-compose.yaml) file (line 21-22), then go to 'Buckets' and create a bucket with the name “src-bucket”.
+Go to http://localhost:9001 and login with the credentials which you can find in the [docker-compose](resources/docker-compose.yaml) file (line 20-21), then go to 'Buckets' and create a bucket with the name “src-bucket”.
 
 ## Upload file to azurite
-Before we upload the file, you have to install Azure CLI. After that, you have to create a blob storage:
+Let`s create a container with the following commands:
 
 ```bash
 conn_str="DefaultEndpointsProtocol=http;AccountName=provider;AccountKey=password;BlobEndpoint=http://127.0.0.1:10000/provider;"
 az storage container create --name src-container --connection-string $conn_str
 ```
 
-If the storage is created successfully, you will get this:
+If the container is created successfully, you will get this:
 ```json
 {
   "created": true
@@ -57,7 +62,7 @@ test-document.txt
 ```
 
 ## Configure the vault
-We already started the vault at the beginning with docker-compose. Now the following commands must be executed in a terminal window to add the necessary secrets.
+We already started the vault at the beginning with docker compose. Now the following commands must be executed in a terminal window to add the necessary secrets.
 
 ```bash
 export VAULT_ADDR='http://0.0.0.0:8200'
@@ -129,10 +134,10 @@ curl -H 'X-Api-Key: password' http://localhost:29193/management/v3/transferproce
 ## Stop docker container
 Execute the following command in a terminal window to stop the docker container:
 ```bash
-docker-compose -f transfer/transfer-05-file-transfer-cloud/resources/docker-compose.yaml down
+docker compose -f transfer/transfer-05-file-transfer-cloud/resources/docker-compose.yaml down
 ```
 
 
 ---
 
-[Previous Chapter](../transfer-04-open-telemetry/README.md)
+[Previous Chapter](../transfer-04-event-consumer/README.md)

transfer/transfer-05-file-transfer-cloud/cloud-transfer-consumer/config.properties

@@ -12,6 +12,4 @@ edc.transfer.proxy.token.verifier.publickey.alias=public-key
 web.http.public.port=29291
 web.http.public.path=/public
 web.http.control.port=29192
-web.http.control.path=/control
-
-edc.converter.usefilesystem=false
+web.http.control.path=/control

transfer/transfer-05-file-transfer-cloud/cloud-transfer-provider/config.properties

@@ -13,9 +13,6 @@ web.http.public.port=19291
 web.http.public.path=/public
 web.http.control.port=19192
 web.http.control.path=/control
-edc.dataplane.api.public.baseurl=http://localhost:19291/public
-
-edc.converter.usefilesystem=false
 
 edc.vault.hashicorp.url=http://127.0.0.1:8200
 edc.vault.hashicorp.token=<root-token>

transfer/transfer-05-file-transfer-cloud/resources/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.9'
 services:
   azurite:
     container_name: azurite