From 7a233cfb1922190d8acbb950e7d3ccede23ca06c Mon Sep 17 00:00:00 2001 From: hamidonos <94196804+hamidonos@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:53:30 +0200 Subject: [PATCH] refactor: building connector jars for telemetry end to end test from docker compose file (#132) * fix: fixing wrong command for ./gradlew test in verify.yaml * fix: raising timeouts to 30s * refactor: building docker images for opentelemetry consumer & provider from dockerfile * fix: remove WEB_HTTP_DSP_PORT & WEB_HTTP_DSP_PATH from docker-compose.yaml * fix: revert temp config * docs: updating README.md for opentelemetry * docs: updating README.md for opentelemetry * refactor: using opentelemetry agent from within docker container --- .github/workflows/verify.yaml | 3 +- .../FileTransferSampleTestCommon.java | 2 +- transfer/transfer-04-open-telemetry/README.md | 36 +++++++++----- .../docker-compose.yaml | 48 ++++++++++--------- .../filetransfer.json | 2 +- .../open-telemetry-consumer/Dockerfile | 11 +++++ .../open-telemetry-consumer/build.gradle.kts | 10 ++-- .../open-telemetry-provider/Dockerfile | 11 +++++ .../open-telemetry-provider/build.gradle.kts | 11 +++-- 9 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile create mode 100644 transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 27141139..b6ee42e3 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,8 +46,7 @@ jobs: - name: End to End Integration Tests uses: ./.github/actions/run-tests with: - command: - - ./gradlew test -DincludeTags="EndToEndTest" + command: ./gradlew test -DincludeTags="EndToEndTest" Upload-Test-Report: needs: diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java index 78e100bb..95e64ec6 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/FileTransferSampleTestCommon.java @@ -52,7 +52,7 @@ public class FileTransferSampleTestCommon { final String sampleAssetFilePath; final File sampleAssetFile; final File destinationFile; - Duration timeout = Duration.ofSeconds(15); + Duration timeout = Duration.ofSeconds(30); Duration pollInterval = Duration.ofMillis(500); String contractNegotiationId; diff --git a/transfer/transfer-04-open-telemetry/README.md b/transfer/transfer-04-open-telemetry/README.md index d7bf1fb8..e35677f8 100644 --- a/transfer/transfer-04-open-telemetry/README.md +++ b/transfer/transfer-04-open-telemetry/README.md @@ -27,7 +27,6 @@ is configured to expose a Prometheus metrics endpoint. To run the consumer, the provider, and Jaeger execute the following commands in the project root folder: ```bash -./gradlew transfer:transfer-04-open-telemetry:open-telemetry-consumer:build transfer:transfer-04-open-telemetry:open-telemetry-provider:build docker-compose -f transfer/transfer-04-open-telemetry/docker-compose.yaml up --abort-on-container-exit ``` @@ -79,26 +78,37 @@ which has to be stored in the root folder of this sample as well. The only addit ```yaml consumer: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/resources + ports: + - "9191:9191" + - "9192:9192" environment: APPLICATIONINSIGHTS_CONNECTION_STRING: APPLICATIONINSIGHTS_ROLE_NAME: consumer # optional: increase log verbosity (default level is INFO) APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL: DEBUG - WEB_HTTP_PORT: 8181 + EDC_HOSTNAME: consumer + OTEL_SERVICE_NAME: consumer + OTEL_TRACES_EXPORTER: jaeger + OTEL_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14250 + OTEL_METRICS_EXPORTER: prometheus + WEB_HTTP_PORT: 9191 WEB_HTTP_PATH: /api - WEB_HTTP_MANAGEMENT_PORT: 8182 + WEB_HTTP_MANAGEMENT_PORT: 9192 WEB_HTTP_MANAGEMENT_PATH: /management - DSP_WEBHOOK_ADDRESS: http://consumer:8181 - volumes: - - ../:/samples - ports: - - 9191:8181 - - 9192:8182 + WEB_HTTP_PROTOCOL_PORT: 9292 + WEB_HTTP_PROTOCOL_PATH: /protocol + EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol + EDC_PARTICIPANT_ID: consumer + EDC_API_AUTH_KEY: password entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/applicationinsights-agent-3.2.8.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar + -javaagent:/resources/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar ``` The Application Insights Java agent will automatically collect metrics from Micrometer, without any configuration needed. diff --git a/transfer/transfer-04-open-telemetry/docker-compose.yaml b/transfer/transfer-04-open-telemetry/docker-compose.yaml index cd7be7fa..e5a99b6b 100644 --- a/transfer/transfer-04-open-telemetry/docker-compose.yaml +++ b/transfer/transfer-04-open-telemetry/docker-compose.yaml @@ -3,7 +3,14 @@ version: "3.8" services: consumer: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile + volumes: + - ./:/resources + ports: + - "9191:9191" + - "9192:9192" environment: EDC_HOSTNAME: consumer OTEL_SERVICE_NAME: consumer @@ -18,21 +25,21 @@ services: WEB_HTTP_PROTOCOL_PATH: /protocol EDC_DSP_CALLBACK_ADDRESS: http://consumer:9292/protocol EDC_PARTICIPANT_ID: consumer - WEB_HTTP_DSP_PORT: 9393 - WEB_HTTP_DSP_PATH: /protocol EDC_API_AUTH_KEY: password - volumes: - - ../:/samples - ports: - - "9191:9191" - - "9192:9192" entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar + -javaagent:/app/libs/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar provider: - image: openjdk:17-jdk-slim-buster + build: + context: ../.. + dockerfile: transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile + volumes: + - ./:/resources + ports: + - "8181:8181" + - "8182:8182" environment: EDC_HOSTNAME: provider OTEL_SERVICE_NAME: provider @@ -46,25 +53,20 @@ services: WEB_HTTP_PROTOCOL_PATH: /protocol EDC_DSP_CALLBACK_ADDRESS: http://provider:8282/protocol EDC_PARTICIPANT_ID: provider - EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /samples/transfer-04-open-telemetry/README.md - volumes: - - ../:/samples - ports: - - "8181:8181" - - "8182:8182" + EDC_SAMPLES_TRANSFER_01_ASSET_PATH: /resources/README.md entrypoint: java - -javaagent:/samples/transfer-04-open-telemetry/opentelemetry-javaagent.jar - -Djava.util.logging.config.file=/samples/transfer-04-open-telemetry/logging.properties - -jar /samples/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar + -javaagent:/app/libs/opentelemetry-javaagent.jar + -Djava.util.logging.config.file=/resources/logging.properties + -jar /app/connector.jar jaeger: image: jaegertracing/all-in-one ports: - - 16686:16686 + - "16686:16686" prometheus: image: prom/prometheus:v2.30.3 volumes: - ./prometheus/:/etc/prometheus/ ports: - - 9090:9090 + - "9090:9090" diff --git a/transfer/transfer-04-open-telemetry/filetransfer.json b/transfer/transfer-04-open-telemetry/filetransfer.json index d0f591e3..b84b7631 100644 --- a/transfer/transfer-04-open-telemetry/filetransfer.json +++ b/transfer/transfer-04-open-telemetry/filetransfer.json @@ -5,7 +5,7 @@ "@type": "TransferRequestDto", "dataDestination": { "type": "File", - "path": "/samples/transfer-04-open-telemetry/README_transferred.md", + "path": "/resources/README_transferred.md", "keyName": "keyName" }, "protocol": "dataspace-protocol-http", diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile new file mode 100644 index 00000000..93680a50 --- /dev/null +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-consumer:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build/libs/consumer.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts index b5c53191..3b466313 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-consumer/build.gradle.kts @@ -24,7 +24,7 @@ plugins { } dependencies { - implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -38,6 +38,7 @@ dependencies { implementation(project(":transfer:transfer-01-file-transfer:status-checker")) + runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.jersey.micrometer) runtimeOnly(libs.edc.jetty.micrometer) runtimeOnly(libs.edc.monitor.jdk.logger) @@ -54,8 +55,7 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" - val file = File(filePath) + val file = file("../opentelemetry-javaagent.jar") if (!file.exists()) { sourceSets["main"] @@ -65,13 +65,13 @@ tasks.register("copyOpenTelemetryJar") { ?.path ?.let { val sourcePath = Paths.get(it) - val targetPath = Paths.get(filePath) + val targetPath = Paths.get(file.path) Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) } } } } -tasks.compileJava { +tasks.build { finalizedBy("copyOpenTelemetryJar") } \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile new file mode 100644 index 00000000..50efc2f5 --- /dev/null +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/Dockerfile @@ -0,0 +1,11 @@ +FROM gradle:jdk17 AS build + +WORKDIR /home/gradle/project/ +COPY --chown=gradle:gradle . /home/gradle/project/ +RUN gradle transfer:transfer-04-open-telemetry:open-telemetry-provider:build + +FROM openjdk:17-slim + +WORKDIR /app +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar /app/libs/opentelemetry-javaagent.jar +COPY --from=build /home/gradle/project/transfer/transfer-04-open-telemetry/open-telemetry-provider/build/libs/provider.jar /app/connector.jar \ No newline at end of file diff --git a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts index e9864bdd..29fc03bf 100644 --- a/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts +++ b/transfer/transfer-04-open-telemetry/open-telemetry-provider/build.gradle.kts @@ -23,7 +23,7 @@ plugins { } dependencies { - implementation(libs.opentelemetry) + implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.core) implementation(libs.edc.data.plane.selector.core) @@ -39,6 +39,8 @@ dependencies { implementation(libs.edc.dsp) implementation(project(":transfer:transfer-01-file-transfer:transfer-file-local")) + + runtimeOnly(libs.opentelemetry) runtimeOnly(libs.edc.monitor.jdk.logger) } @@ -53,8 +55,7 @@ tasks.withType { tasks.register("copyOpenTelemetryJar") { doLast { - val filePath = "transfer/transfer-04-open-telemetry/opentelemetry-javaagent.jar" - val file = File(filePath) + val file = file("../opentelemetry-javaagent.jar") if (!file.exists()) { sourceSets["main"] @@ -64,13 +65,13 @@ tasks.register("copyOpenTelemetryJar") { ?.path ?.let { val sourcePath = Paths.get(it) - val targetPath = Paths.get(filePath) + val targetPath = Paths.get(file.path) Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) } } } } -tasks.compileJava { +tasks.build { finalizedBy("copyOpenTelemetryJar") } \ No newline at end of file