From 2ea27686b1938b272f63f12217eff264206e4a57 Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Fri, 6 Oct 2023 08:43:31 +0200 Subject: [PATCH] fix: #86 sample 04 terminating on startup (#128) * fix: working sample 04 * test: added test for docker compose sample * style: cosmetic changes * test: temporarily disable progress checking of transfer * docs: added explanation for commented line * test: download OpenTelemetry Jar before test execution * gradle: download OpenTelemetry before build step * removing container name from prometheus as it's not supported by testcontainers * fixed typo in libs.versions.toml * copy opentelemetry jar from gradle cache * fixed default sample tests for sample 04 * fixed build process * asserting Jaeger state OK in telemetry test * fix: fixing wrong testcontainer reference in .toml file * fix: running gradle build before test * fix: removing gradle build before test for github action --------- Co-authored-by: Matthias De Geyter --- .github/workflows/verify.yaml | 3 +- gradle/libs.versions.toml | 10 ++- system-tests/build.gradle.kts | 3 + .../FileTransferSampleTestCommon.java | 17 +++- .../transfer/Transfer04openTelemetryTest.java | 84 +++++++++++++++++++ transfer/transfer-04-open-telemetry/README.md | 7 +- .../contractoffer.json | 47 ++++------- .../docker-compose.yaml | 17 ++-- .../filetransfer.json | 21 ++--- .../transfer-04-open-telemetry/input-file.txt | 1 - .../open-telemetry-consumer/build.gradle.kts | 38 ++++++++- .../open-telemetry-provider/build.gradle.kts | 31 +++++++ 12 files changed, 212 insertions(+), 67 deletions(-) create mode 100644 system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java delete mode 100644 transfer/transfer-04-open-telemetry/input-file.txt diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b6ee42e3..27141139 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,7 +46,8 @@ jobs: - name: End to End Integration Tests uses: ./.github/actions/run-tests with: - command: ./gradlew test -DincludeTags="EndToEndTest" + command: + - ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 391544d0..d467c6a0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,9 +12,8 @@ okhttp-mockwebserver = "5.0.0-alpha.11" openTelemetry = "1.18.0" restAssured = "5.3.2" rsApi = "3.1.0" +testcontainers = "1.19.1" kafkaClients = "3.6.0" -testContainers = "1.19.1" - [libraries] assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } @@ -69,9 +68,12 @@ junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "jun okhttp-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp-mockwebserver" } opentelemetry-annotations = { module = "io.opentelemetry:opentelemetry-extension-annotations", version.ref = "openTelemetry" } restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafkaClients" } -testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testContainers" } -testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testContainers" } +testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } +testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } +opentelemetry = "io.opentelemetry.javaagent:opentelemetry-javaagent:1.30.0" [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index c5276dfa..8d033245 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.okhttp.mockwebserver) testImplementation(libs.restAssured) + testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.junit) testImplementation(libs.testcontainers.kafka) testImplementation(libs.kafka.clients) @@ -36,6 +37,8 @@ dependencies { testCompileOnly(project(":transfer:transfer-01-file-transfer:file-transfer-provider")) testCompileOnly(project(":transfer:transfer-02-file-transfer-listener:file-transfer-listener-consumer")) testCompileOnly(project(":transfer:transfer-03-modify-transferprocess:modify-transferprocess-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-consumer")) + testCompileOnly(project(":transfer:transfer-04-open-telemetry:open-telemetry-provider")) testCompileOnly(project(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")) testCompileOnly(project(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")) } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index be7d0474..78e100bb 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon { final String sampleAssetFilePath; final File sampleAssetFile; final File destinationFile; - Duration timeout = Duration.ofSeconds(30); + Duration timeout = Duration.ofSeconds(15); Duration pollInterval = Duration.ofMillis(500); String contractNegotiationId; @@ -133,10 +133,15 @@ void assertTransferProcessStatusConsumerSide(String transferProcessId) { * This method corresponds to the command in the sample: {@code curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: password" -d @transfer/transfer-01-file-transfer/contractoffer.json "http://localhost:9192/management/v2/contractnegotiations"} */ void initiateContractNegotiation() { + initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + } + + + void initiateContractNegotiation(String contractOfferFilePath) { contractNegotiationId = given() .headers(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) .contentType(ContentType.JSON) - .body(new File(TestUtils.findBuildRoot(), CONTRACT_OFFER_FILE_PATH)) + .body(new File(TestUtils.findBuildRoot(), contractOfferFilePath)) .when() .post(MANAGEMENT_API_URL + "/v2/contractnegotiations") .then() @@ -181,8 +186,8 @@ void lookUpContractAgreementId() { * * @throws IOException Thrown if there was an error accessing the transfer request file defined in {@link FileTransferSampleTestCommon#TRANSFER_FILE_PATH}. */ - String requestTransferFile() throws IOException { - var transferJsonFile = getFileFromRelativePath(TRANSFER_FILE_PATH); + String requestTransferFile(String transferFilePath) throws IOException { + var transferJsonFile = getFileFromRelativePath(transferFilePath); var requestBody = readAndUpdateDataRequestFromJsonFile(transferJsonFile, contractAgreementId); var jsonPath = given() @@ -205,6 +210,10 @@ String requestTransferFile() throws IOException { return transferProcessId; } + String requestTransferFile() throws IOException { + return requestTransferFile(TRANSFER_FILE_PATH); + } + /** * Reads a transfer request file with changed value for contract agreement ID and file destination path. * diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java new file mode 100644 index 00000000..eb3d7029 --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer04openTelemetryTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Mercedes-Benz Tech Innovation GmbH - Sample workflow test + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.junit.ClassRule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +@EndToEndTest +@Testcontainers +public class Transfer04openTelemetryTest { + + private static final String SAMPLE_FOLDER = "transfer/transfer-04-open-telemetry"; + private static final String DOCKER_COMPOSE_YAML = "/docker-compose.yaml"; + private static final String SAMPLE_ASSET_FILE_PATH = SAMPLE_FOLDER + "/README.md"; + private static final String DESTINATION_FILE_PATH = SAMPLE_FOLDER + "/README_transferred.md"; + private static final String CONTRACT_OFFER_FILE_PATH = SAMPLE_FOLDER + "/contractoffer.json"; + private static final String FILE_TRANSFER_FILE_PATH = SAMPLE_FOLDER + "/filetransfer.json"; + private static final String JAEGER_URL = "http://localhost:16686"; + + private final FileTransferSampleTestCommon testUtils = new FileTransferSampleTestCommon(SAMPLE_ASSET_FILE_PATH, DESTINATION_FILE_PATH); + + @ClassRule + public static DockerComposeContainer environment = + new DockerComposeContainer(FileTransferSampleTestCommon.getFileFromRelativePath(SAMPLE_FOLDER + DOCKER_COMPOSE_YAML)) + .withLocalCompose(true) + .waitingFor("consumer", Wait.forLogMessage(".*ready.*", 1)); + + @BeforeAll + static void setUp() { + environment.start(); + } + + @Test + void runSampleSteps() throws Exception { + testUtils.assertTestPrerequisites(); + testUtils.initiateContractNegotiation(CONTRACT_OFFER_FILE_PATH); + testUtils.lookUpContractAgreementId(); + var transferProcessId = testUtils.requestTransferFile(FILE_TRANSFER_FILE_PATH); + testUtils.assertDestinationFileContent(); + testUtils.assertTransferProcessStatusConsumerSide(transferProcessId); + assertJaegerState(); + } + + private void assertJaegerState() { + try { + var url = new URL(JAEGER_URL); + var huc = (HttpURLConnection) url.openConnection(); + assertThat(huc.getResponseCode()).isEqualTo(HttpStatus.SC_OK); + } catch (IOException e) { + fail("Unable to assert Jaeger state", e); + } + } + + @AfterEach + protected void tearDown() { + testUtils.cleanTemporaryTestFiles(); + } +} diff --git a/transfer/transfer-04-open-telemetry/README.md b/transfer/transfer-04-open-telemetry/README.md index 77040306..d7bf1fb8 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/transfer/transfer-04-open-telemetry/README.md @@ -12,12 +12,7 @@ several popular [libraries and frameworks](https://github.com/open-telemetry/ope In order to visualize and analyze the traces and metrics, we use [OpenTelemetry exporters](https://opentelemetry.io/docs/instrumentation/js/exporters/) to export data into the Jaeger -tracing backend and a Prometheus endpoint. - -## Prerequisites - -Download the [opentelemetry-javaagent.jar](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar) -and place it in the root folder of this sample. +tracing backend and a Prometheus endpoint. ## Run the sample diff --git a/transfer/transfer-04-open-telemetry/contractoffer.json b/transfer/transfer-04-open-telemetry/contractoffer.json index db928391..5baecd25 100644 --- a/transfer/transfer-04-open-telemetry/contractoffer.json +++ b/transfer/transfer-04-open-telemetry/contractoffer.json @@ -1,43 +1,24 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "NegotiationInitiateRequestDto", "connectorId": "provider", + "consumerId": "consumer", + "providerId": "provider", "connectorAddress": "http://provider:8282/protocol", "protocol": "dataspace-protocol-http", "offer": { - "offerId": "1:3a75736e-001d-4364-8bd4-9888490edb58", + "offerId": "1:test-document:3a75736e-001d-4364-8bd4-9888490edb58", "assetId": "test-document", "policy": { - "uid": "956e172f-2de1-4501-8881-057a57fd0e69", - "permissions": [ - { - "edctype": "dataspaceconnector:permission", - "uid": null, - "target": "test-document", - "action": { - "type": "USE", - "includedIn": null, - "constraint": null - }, - "assignee": null, - "assigner": null, - "constraints": [], - "duties": [] - } - ], - "prohibitions": [], - "obligations": [], - "extensibleProperties": {}, - "inheritsFrom": null, - "assigner": null, - "assignee": null, - "target": null, - "@type": { - "@policytype": "set" - } - }, - "asset": { - "properties": { - "asset:prop:id": "test-document" - } + "@id": "1:test-document:13dce0f1-52ed-4554-a194-e83e92733ee5", + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": "test-document" } } } diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index ec4bae44..cd7be7fa 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -5,6 +5,7 @@ services: consumer: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: consumer OTEL_SERVICE_NAME: consumer OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -13,9 +14,12 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - WEB_HTTP_DSP_PORT: 9292 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer + WEB_HTTP_DSP_PORT: 9393 WEB_HTTP_DSP_PATH: /protocol - DSP_WEBHOOK_ADDRESS: http://consumer:9292 EDC_API_AUTH_KEY: password volumes: - ../:/samples @@ -30,6 +34,7 @@ services: provider: image: openjdk:17-jdk-slim-buster environment: + EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider OTEL_TRACES_EXPORTER: jaeger OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 @@ -37,8 +42,11 @@ services: WEB_HTTP_PATH: /api WEB_HTTP_MANAGEMENT_PORT: 8182 WEB_HTTP_MANAGEMENT_PATH: /management - DSP_WEBHOOK_ADDRESS: http://provider:8282 - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/input-file.txt + WEB_HTTP_PROTOCOL_PORT: 8282 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol + EDC_PARTICIPANT_ID: provider + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md volumes: - ../:/samples ports: @@ -56,7 +64,6 @@ services: prometheus: image: prom/prometheus:v2.30.3 - container_name: prometheus volumes: - ./prometheus/:/etc/prometheus/ ports: diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index 3a2bfba1..d0f591e3 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -1,16 +1,17 @@ { + "@context": { + "edc": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "dataDestination": { + "type": "File", + "path": "/samples/transfer-04-open-telemetry/README_transferred.md", + "keyName": "keyName" + }, "protocol": "dataspace-protocol-http", "assetId": "test-document", "contractId": "{agreement ID}", - "dataDestination": { - "path": "/samples/output-file.txt", - "keyName": "keyName", - "type": "File" - }, - "transferType": { - "contentType": "application/octet-stream", - "isFinite": true - }, + "connectorId": "provider", "connectorAddress": "http://provider:8282/protocol", - "connectorId": "consumer" + "privateProperties": {} } diff --git a/transfer/transfer-04-open-telemetry/input-file.txt b/transfer/transfer-04-open-telemetry/input-file.txt deleted file mode 100644 index b4f8b365..00000000 --- a/transfer/transfer-04-open-telemetry/input-file.txt +++ /dev/null @@ -1 +0,0 @@ -This is a sample file \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index ade9456d..b5c53191 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -20,16 +24,20 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.micrometer.core) - + implementation(libs.edc.api.observability) implementation(libs.edc.configuration.filesystem) implementation(libs.edc.iam.mock) - implementation(libs.edc.auth.tokenbased) implementation(libs.edc.management.api) - implementation(libs.edc.dsp) + + implementation(project(":transfer:transfer-01-file-transfer:status-checker")) + runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) @@ -43,3 +51,27 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("consumer.jar") } + +tasks.register("copyOpenTelemetryJar") { + doLast { + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } + } +} + +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") +} \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index 34930dd8..e9864bdd 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -1,3 +1,7 @@ +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + /* * Copyright (c) 2022 Microsoft Corporation * @@ -19,7 +23,10 @@ plugins { } dependencies { + implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) + implementation(libs.edc.data.plane.selector.core) implementation(libs.edc.api.observability) @@ -43,3 +50,27 @@ tasks.withType { mergeServiceFiles() archiveFileName.set("provider.jar") } + +tasks.register("copyOpenTelemetryJar") { + doLast { + val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" + val file = File(filePath) + + if (!file.exists()) { + sourceSets["main"] + .runtimeClasspath + .files + .find { it.name.contains("opentelemetry-javaagent") } + ?.path + ?.let { + val sourcePath = Paths.get(it) + val targetPath = Paths.get(filePath) + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) + } + } + } +} + +tasks.compileJava { + finalizedBy("copyOpenTelemetryJar") +} \ No newline at end of file