diff --git a/.github/workflows/0.build-base-nifi-docker.yml b/.github/workflows/0.build-base-nifi-docker.yml index 0443cd9d1..710ec06a8 100644 --- a/.github/workflows/0.build-base-nifi-docker.yml +++ b/.github/workflows/0.build-base-nifi-docker.yml @@ -4,12 +4,12 @@ on: workflow_dispatch: env: - BASE_IMAGE_NAME: openjdk - BASE_IMAGE_TAG: 18-ea-bullseye + BASE_IMAGE_NAME: amazoncorretto + BASE_IMAGE_TAG: 21-alpine-jdk NIFI_VERSION: 1.20.0 DISTRO_PATH: 1.20.0 NIFI_DOCKER_IMAGE_NAME: ldes/nifi - NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk18 + NIFI_DOCKER_IMAGE_VERSION: 1.20.0-jdk21 jobs: job-build-push-docker-nifi: @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -27,13 +27,13 @@ jobs: with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build NiFi JDK18 Docker image + - name: Build NiFi JDK21 Docker image run: | echo ENVIRONMENT echo BASE_IMAGE_NAME: ${BASE_IMAGE_NAME} echo BASE_IMAGE_TAG: ${BASE_IMAGE_TAG} echo NIFI_VERSION: ${NIFI_VERSION} - echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk18 + echo TARGET_IMAGE_TAG: ${NIFI_VERSION}-jdk21 echo NIFI_DOCKER_IMAGE_NAME: ${NIFI_DOCKER_IMAGE_NAME} echo NIFI_DOCKER_IMAGE_VERSION: ${NIFI_DOCKER_IMAGE_VERSION} curl -L https://github.com/apache/nifi/archive/refs/tags/rel/nifi-${NIFI_VERSION}.tar.gz | tar -zxv --strip-components=2 nifi-rel-nifi-${NIFI_VERSION}/nifi-docker/dockerhub/ diff --git a/.github/workflows/1.a-build-pr.yml b/.github/workflows/1.a-build-pr.yml index 6b4702ef0..6fd6f9171 100644 --- a/.github/workflows/1.a-build-pr.yml +++ b/.github/workflows/1.a-build-pr.yml @@ -17,14 +17,14 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: '18' - distribution: 'adopt' + java-version: '21' + distribution: 'zulu' - name: Cache Maven packages uses: actions/cache@v1 with: diff --git a/.github/workflows/2.a-build-main.yml b/.github/workflows/2.a-build-main.yml index db2c1ffbe..ce6f4354d 100644 --- a/.github/workflows/2.a-build-main.yml +++ b/.github/workflows/2.a-build-main.yml @@ -23,12 +23,12 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v3 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Cache SonarCloud packages uses: actions/cache@v1 with: @@ -42,10 +42,10 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: '18' - distribution: 'adopt' + java-version: '21' + distribution: 'zulu' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD diff --git a/.github/workflows/2.b-build-ldes-discoverer-docker.yml b/.github/workflows/2.b-build-ldes-discoverer-docker.yml index 6bf687b49..e9f8fcd5d 100644 --- a/.github/workflows/2.b-build-ldes-discoverer-docker.yml +++ b/.github/workflows/2.b-build-ldes-discoverer-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/2.b-build-ldio-docker.yml b/.github/workflows/2.b-build-ldio-docker.yml index 92cd1e90e..2fea978e0 100644 --- a/.github/workflows/2.b-build-ldio-docker.yml +++ b/.github/workflows/2.b-build-ldio-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/2.b-build-nifi-workbench-docker.yml b/.github/workflows/2.b-build-nifi-workbench-docker.yml index 8c0587e12..08c620edd 100644 --- a/.github/workflows/2.b-build-nifi-workbench-docker.yml +++ b/.github/workflows/2.b-build-nifi-workbench-docker.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -30,11 +30,11 @@ jobs: # See: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381 driver-opts: | image=moby/buildkit:v0.10.6 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Determine version run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/.github/workflows/3.approve-snapshot.yml b/.github/workflows/3.approve-snapshot.yml index ba181ae13..21b8f2752 100644 --- a/.github/workflows/3.approve-snapshot.yml +++ b/.github/workflows/3.approve-snapshot.yml @@ -14,12 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 - - name: Set up JDK 18 - uses: actions/setup-java@v3 + uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: distribution: zulu - java-version: 18 + java-version: 21 - name: Define docker variables run: | export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) diff --git a/CODEOWNERS b/CODEOWNERS index bbcc1dd2d..4bdfa35f0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1 @@ -* @Yalz @rorlic @pj-cegeka @Tomvbe @jobulcke - +* @Yalz @rorlic @pj-cegeka @jobulcke @ferre-vaes diff --git a/docs/_ldio/examples/ex2-scrape-api.md b/docs/_ldio/examples/ex2-scrape-api.md index 2f29fe30f..89e8dacb9 100644 --- a/docs/_ldio/examples/ex2-scrape-api.md +++ b/docs/_ldio/examples/ex2-scrape-api.md @@ -41,7 +41,7 @@ mappings: - [a, cs:GameDeal] - [cs:title, $(title)] - [cs:metacriticLink, $(metacriticLink)] - - [cs:thumb, $(thumb)~iri] + - [cs:thumb, $(thumb)] - p: cs:releaseDate o: function: ldi:epochToIso8601 diff --git a/docs/_ldio/examples/ex2/ex2-mapping.ttl b/docs/_ldio/examples/ex2/ex2-mapping.ttl index ffa334a33..0cf19adc2 100644 --- a/docs/_ldio/examples/ex2/ex2-mapping.ttl +++ b/docs/_ldio/examples/ex2/ex2-mapping.ttl @@ -39,7 +39,7 @@ map:om_002 rml:reference "metacriticLink" ; map:om_003 rml:reference "thumb" ; rdf:type rr:ObjectMap ; - rr:termType rr:IRI . + rr:termType rr:Literal . map:om_004 map:fn_000 ; rdf:type ; diff --git a/docs/_ldio/ldio-adapters/index.md b/docs/_ldio/ldio-adapters/index.md index 2fc0c4ece..7ac9edad2 100644 --- a/docs/_ldio/ldio-adapters/index.md +++ b/docs/_ldio/ldio-adapters/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 4 --- -# Linked Data Adapters +# Linked Data Orchestrator Adapters The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-core/index.md b/docs/_ldio/ldio-core/index.md index 09e859890..06697a9cb 100644 --- a/docs/_ldio/ldio-core/index.md +++ b/docs/_ldio/ldio-core/index.md @@ -6,6 +6,6 @@ has_toc: true nav_order: 2 --- -# LDIO Core +# Linked Data Orchestrator Core Contains common components used by multiple other LDIO components diff --git a/docs/_ldio/ldio-inputs/index.md b/docs/_ldio/ldio-inputs/index.md index 7f1884aa4..6c6fa668b 100644 --- a/docs/_ldio/ldio-inputs/index.md +++ b/docs/_ldio/ldio-inputs/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 3 --- -# LDIO Inputs +# Linked Data Orchestrator Inputs The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md index ec9d52f5c..43963891a 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client-connector.md @@ -24,7 +24,7 @@ This component exposes two endpoints: transfer with the connector and also keep the transfer alive while consuming the LDES (e.g. request a new token when it expires). -3. http://://token +2. http://://token This endpoint should never be called directly. This is the callback to be provided in the transfer request. The EDC connector will use this callback endpoint to provide the LDES Client with a token. @@ -54,6 +54,12 @@ input: source-format: application/n-quads ``` +## INIT phase + +Contrary to the other ldio-input components, the connector waits in the INIT status for the edc-token and will only progress to the RUNNING status once it has received this token. +More on the statuses in ldio can be found [here.](../pipeline-management/pipeline-status) + + ## Pausing When paused, this component will stop processing the current fragment and not make any calls to the server. diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index 6a70df5fd..182f287b0 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -14,14 +14,14 @@ An LDIO wrapper component for the [LDI LDES Client building block](../../core/ld ### General properties -| Property | Description | Required | Default | Example | Supported values | -|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:--------------------------------------------------------------| -| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | +| Property | Description | Required | Default | Example | Supported values | +|:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | | source-format | The 'Content-Type' that should be requested to the server. | No | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | -| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | -| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | -| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | -| enable-exactly-once | Indicates whether member must be send exactly once or at least once | No | true | true | true or false | +| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | +| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | +| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | +| enable-exactly-once | Indicates whether member must be send exactly once or at least once | No | true | true | true or false | > **_NOTE:_** The default `source-format` is `text/turtle`, as this rdf format supports relative uri's. However, if > relative uri's are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better diff --git a/docs/_ldio/ldio-outputs/index.md b/docs/_ldio/ldio-outputs/index.md index d21d8dd5f..91d002f85 100644 --- a/docs/_ldio/ldio-outputs/index.md +++ b/docs/_ldio/ldio-outputs/index.md @@ -6,7 +6,7 @@ has_toc: true nav_order: 6 --- -# Linked Data Adapters +# Linked Data Orchestrator Outputs The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. diff --git a/docs/_ldio/ldio-transformers/index.md b/docs/_ldio/ldio-transformers/index.md index cb1103a0d..663022c6a 100644 --- a/docs/_ldio/ldio-transformers/index.md +++ b/docs/_ldio/ldio-transformers/index.md @@ -6,8 +6,4 @@ has_toc: true nav_order: 5 --- -# Linked Data Adapters - -The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. - -Each component can be wrapped in a desired implementation framework (LDI-orchestrator, NiFi, ...) to be used. +# Linked Data Orchestrator Transformers \ No newline at end of file diff --git a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md index c4031f18d..51f6ce1a0 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md +++ b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md @@ -12,13 +12,13 @@ An LDIO wrapper component for the [LDI Version Object Creator building block](.. ## Config -| Property | Description | Required | Default | Example | Supported values | -|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:--------------------------------------------------------------------|:-----------------| -| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No | Current Timestamp | | String | -| member-type | Defines the RDF type of the version object | No | N/A | https://uri.etsi.org/ngsi-ld/default-context/Device | String | -| delimiter | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved) | No | / | / | String | -| generatedAt-property | If defined, a statement will be added to the model with the observedAt value and the given property. | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | String | -| versionOf-property | If defined, a statement will be added to the model with the versionOf value and the given property. | No | N/A | http://purl.org/dc/terms/isVersionOf | String | +| Property | Description | Required | Default | Example | Supported values | +|:-----------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------|:------------------------------------------|:-----------------| +| date-observed-property | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No | Current Timestamp | | String | +| member-type | Defines the RDF type of the version object | No | N/A | https://example.org/Person | String | +| delimiter | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved) | No | / | / | String | +| generatedAt-property | If defined, a statement will be added to the model with the observedAt value and the given property. | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | String | +| versionOf-property | If defined, a statement will be added to the model with the versionOf value and the given property. | No | N/A | http://purl.org/dc/terms/isVersionOf | String | A property path can be provided for the date-observed-property. For example for the following ttl: diff --git a/docs/_ldio/pipeline-management/pipeline-status.md b/docs/_ldio/pipeline-management/pipeline-status.md new file mode 100644 index 000000000..35c0a0020 --- /dev/null +++ b/docs/_ldio/pipeline-management/pipeline-status.md @@ -0,0 +1,54 @@ +--- +layout: default +parent: Pipeline Management +title: Pipeline Status +--- +# Status + +An individual ldio-pipeline can be in one of several different statuses. +These different statuses and their behaviour are dependent on the input component of the pipeline. + +# Overview Of The Status Flow + + +
+graph LR; + INIT --> RUNNING; + INIT --> STOPPED; + RUNNING --> STOPPED; + RUNNING --> HALTED; + HALTED --> RUNNING; + HALTED --> STOPPED; +
+ +The above diagram shows the flow between the different statuses of the pipeline. + +## INIT + +The startup step of the ldio pipeline. +This is the preparation step before the input component can receive data and pass it on to the rest of the pipeline. +In most components this step will take little time. The only exception for now is [the client-connector](../ldio-inputs/ldio-ldes-client-connector). +It is not possible to pause the pipeline while in this state. + +## RUNNING + +This status indicates that the ldio pipeline is running and that the input component is ready to or currently receiving or fetching data. +This is the only state in which the pipeline can be paused. + +## HALTED + +Every ldio pipeline can be paused, the exact behavior of which depends on the ldio input component used. +A more in depth explanation can be found on the pages for [the individual input components.](../ldio-inputs/index) +Currently, the HALTED status can only be reached through manually pausing the pipeline through [the pipeline-api](./pipeline-api). + +## STOPPED + +When a pipeline is deleted, it will first change to the STOPPED status, this ensures the state is correctly saved in stateful components and that the entire pipeline can be gracefully shutdown. +This status will only last a short while before the pipeline is deleted and the status can no longer be queried. +The pipeline can be stopped from any other status. + +## Status Change Source + +The ldio pipeline keeps track of if the last status change was triggered manually or automatic. +Manually changing the status can be done through [the pipeline-api](./pipeline-api). +When in the RUNNING state, this indicates if the pipeline has been started (automatic) or unpaused (manual). \ No newline at end of file diff --git a/ldes-discoverer.Dockerfile b/ldes-discoverer.Dockerfile index f31bb610a..1e96b2019 100644 --- a/ldes-discoverer.Dockerfile +++ b/ldes-discoverer.Dockerfile @@ -3,7 +3,7 @@ # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage @@ -13,7 +13,7 @@ RUN mvn clean install -DskipTests # # RUN THE APPLICATION # -FROM amazoncorretto:17-alpine-jdk +FROM amazoncorretto:21-alpine-jdk RUN adduser -D -u 2000 ldes-discoverer USER ldes-discoverer diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 7990675fb..83d6474d6 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 82fdf9517..d8461781b 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 4baa82451..67e492312 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 5cd11c80e..500dde2a9 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 071f50cc2..ae20022db 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 2bf57b7ae..6e699a8a3 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java index 4847e48f6..abf26568d 100644 --- a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java +++ b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeFinderSteps.java @@ -15,7 +15,7 @@ public class StartingTreeNodeFinderSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private StartingTreeNodeFinder startingTreeNodeFinder; private StartingNodeRequest startingNodeRequest; diff --git a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java index 30c339dc6..6b3e48958 100644 --- a/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java +++ b/ldi-core/ldes-client/starting-node/src/test/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinderSteps.java @@ -22,7 +22,7 @@ public class StartingTreeNodeRelationsFinderSteps { @Given("I have a StartingTreeNodeRelationsFinder") public void iHaveAStartingTreeNodeRelationsFinder() { - final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); final RequestExecutor requestExecutor = requestExecutorFactory.createNoAuthExecutor(); startingTreeNodeRelationsFinder = new StartingTreeNodeRelationsFinder(requestExecutor); } diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index 45a2529e1..9e71cfc12 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java index 8284671e4..846fd0c5b 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/test/java/ldes/client/treenodefetcher/TreeNodeFetcherSteps.java @@ -15,7 +15,7 @@ public class TreeNodeFetcherSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeFetcher treeNodeFetcher; private TreeNodeRequest treeNodeRequest; private TreeNodeResponse treeNodeResponse; diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index 25c3d6819..64e3ec1f7 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java index 3ef50e8c7..1578ed813 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java @@ -12,7 +12,6 @@ public class LdesStructureDiscoverer { private static final Logger log = LoggerFactory.getLogger(LdesStructureDiscoverer.class); - private final TreeRelationsFetcher treeNodeFetcher; private final String startingUrl; private final Lang sourceFormat; private final RequestExecutor requestExecutor; @@ -21,7 +20,6 @@ public LdesStructureDiscoverer(String startingUrl, Lang sourceFormat, RequestExe this.startingUrl = startingUrl; this.sourceFormat = sourceFormat; this.requestExecutor = requestExecutor; - this.treeNodeFetcher = new TreeRelationsFetcher(requestExecutor); } public LdesStructure discoverLdesStructure() { @@ -47,7 +45,7 @@ private List getStartingTreeRelations() { private void fetchRelations(TreeRelation relation, boolean isChildOfRoot) { final TreeNodeRequest treeNodeRequest = new TreeNodeRequest(relation.getUri(), sourceFormat); try { - treeNodeFetcher.fetchTreeRelations(treeNodeRequest) + new TreeRelationsFetcher(requestExecutor).fetchTreeRelations(treeNodeRequest) .parallelStream() .forEach(fetchedRelation -> { if (fetchedRelation.isRequired() || isChildOfRoot) { diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java index da3f4f23b..893acde45 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererSteps.java @@ -17,7 +17,7 @@ public class LdesStructureDiscovererSteps { public void iHaveATreeDiscoverer() { final String startingUrl = "http://localhost:10101/200-first-4-relations"; final Lang sourceFormat = Lang.TURTLE; - ldesStructureDiscoverer = new LdesStructureDiscoverer(startingUrl, sourceFormat, new RequestExecutorFactory().createNoAuthExecutor()); + ldesStructureDiscoverer = new LdesStructureDiscoverer(startingUrl, sourceFormat, new RequestExecutorFactory(false).createNoAuthExecutor()); } @When("I start discovering the tree node relations") diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java index 9ef3dbfff..f606fb1b3 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscovererTest.java @@ -30,7 +30,7 @@ class LdesStructureDiscovererTest { @BeforeEach void setUp() { - final RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + final RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); discoverer = new LdesStructureDiscoverer("http://localhost:%d%s".formatted(WIREMOCK_PORT, STARTING_ENDPOINT), Lang.TURTLE, requestExecutor); } diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java index 10efb39bd..aac9ccfac 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/test/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcherSteps.java @@ -15,7 +15,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TreeRelationsFetcherSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeRequest treeNodeRequest; private TreeRelationsFetcher treeRelationsFetcher; private List treeNodeRelations; diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index d0abf79ad..1f680fabc 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java index 3cc5c2bf1..70611eb6a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberRepository.java @@ -20,7 +20,7 @@ public Optional getTreeMember() { @Override public void deleteMember(MemberRecord member) { - members.remove(); + members.poll(); } @Override diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java index da33f41a7..df0be3d76 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java @@ -23,7 +23,7 @@ class TreeNodeProcessorFactory { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, List url, Lang sourceFormat) { final LdesMetaData ldesMetaData = new LdesMetaData(url, sourceFormat); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java index 378fe7188..1f870faf4 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java @@ -30,7 +30,7 @@ import static org.junit.jupiter.api.Assertions.*; public class MemberSupplierSteps { - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeProcessor treeNodeProcessor; private TreeNodeRecordRepository treeNodeRecordRepository; private MemberRepository memberRepository; diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index e5ee8f5c0..5c97d2af2 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldi-common diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 9c201b3f8..10972eeb1 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index f8200b075..bc8cdbd86 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 @@ -30,7 +30,6 @@ 5.9.2 4.2.0 - 1.12.23 7.13.0 1.9.3 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index cb72e0e49..0295d1d0c 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index eef320836..6c03010c6 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT @@ -59,10 +59,6 @@ org.eclipse.rdf4j rdf4j-rio-trix - - org.eclipse.rdf4j - rdf4j-rio-turtle - org.eclipse.rdf4j rdf4j-rio-n3 diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java index aec100cc2..9ea379a4c 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java @@ -6,9 +6,12 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.MaterialiserConnection; import org.apache.jena.rdf.model.Model; import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -32,8 +35,7 @@ public void process(List jenaModels) { synchronized (materialiserConnection) { try { jenaModels.stream().map(JenaToRDF4JConverter::convert).forEach(updateModel -> { - Set entityIds = ModelSubjectsExtractor.extractSubjects(updateModel); - deleteEntities(entityIds); + deleteEntity(updateModel); materialiserConnection.add(updateModel); }); materialiserConnection.commit(); @@ -48,41 +50,36 @@ public CompletableFuture processAsync(List jenaModels) { return CompletableFuture.runAsync(() -> process(jenaModels)); } + public void closeConnection() { + materialiserConnection.close(); + } + public void shutdown() { materialiserConnection.shutdown(); } - protected void deleteEntities(Set subjects) { - subjects.forEach(this::deleteEntity); + protected void deleteEntity(org.eclipse.rdf4j.model.Model model) { + getAllSubjectsFromModel(model) + .forEach(subject -> materialiserConnection.remove(subject, null, null)); } - protected void deleteEntity(Resource subject) { - final String query = """ - DELETE { - ?s ?p ?o . - ?blankNode ?blankNodePredicate ?blankNodeObject . - } - WHERE { - ?s ?p ?o . - FILTER(?s = <%s>) + private Set getAllSubjectsFromModel(org.eclipse.rdf4j.model.Model model) { + final Set subjects = ModelSubjectsExtractor.extractSubjects(model); + final Deque subjectStack = new ArrayDeque<>(subjects); - OPTIONAL { - { - ?o ?blankNodePredicate ?blankNodeObject . - FILTER(isBlank(?o)) - BIND(?o AS ?blankNode) - } - UNION - { - ?o ?nestedPredicate ?nestedObject . - FILTER(isBlank(?nestedObject)) - ?nestedObject ?blankNodePredicate ?blankNodeObject . - BIND(?nestedObject AS ?blankNode) - } - } - } - """.formatted(subject.stringValue()); + while (!subjectStack.isEmpty()) { + Resource subject = subjectStack.pop(); - materialiserConnection.executeUpdate(query); + materialiserConnection.getStatements(subject, null, null).forEach(statement -> { + Value object = statement.getObject(); + if (object.isBNode()) { + Resource bnode = (Resource) object; + subjectStack.push(bnode); + subjects.add(bnode); + } + }); + + } + return subjects; } } diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java index e7d6fcd01..ae20817b3 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java @@ -1,10 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.services; +import org.apache.jena.rdf.model.Literal; import org.apache.jena.rdf.model.RDFNode; -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.model.impl.LinkedHashModel; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; @@ -43,7 +41,11 @@ private static Value convertValue(RDFNode node) { org.apache.jena.rdf.model.Resource resource = (org.apache.jena.rdf.model.Resource) node; return convertResource(resource); } else { - return SimpleValueFactory.getInstance().createLiteral(node.toString()); + Literal literal = node.asLiteral(); + if(literal.getLanguage() != null && !literal.getLanguage().isEmpty()) { + return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), literal.getLanguage()); + } + return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), SimpleValueFactory.getInstance().createIRI(literal.getDatatypeURI())); } } } diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java index 715240d00..48854ff58 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java @@ -6,6 +6,8 @@ import org.eclipse.rdf4j.repository.RepositoryResult; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import java.util.Optional; + public class MaterialiserConnection { private final String namedGraph; private final RepositoryConnectionHolder holder; @@ -15,22 +17,23 @@ public MaterialiserConnection(RepositoryManager repositoryManager, String reposi this.holder = new RepositoryConnectionHolder(repositoryManager, repositoryId); } - public void add(Model model) { - if (namedGraph != null && !namedGraph.isEmpty()) { - var namedGraphIRI = SimpleValueFactory.getInstance().createIRI(namedGraph); - holder.getConnection().add(model, namedGraphIRI); - } else { - holder.getConnection().add(model); - } + getNamedGraphIri().ifPresentOrElse( + namedGraphIri -> holder.getConnection().add(model, namedGraphIri), + () -> holder.getConnection().add(model)); } public RepositoryResult getStatements(Resource subject, IRI predicate, Value object) { - return holder.getConnection().getStatements(subject, predicate, object); + return getNamedGraphIri() + .map(namedGraphIri -> holder.getConnection().getStatements(subject, predicate, object, namedGraphIri)) + .orElseGet(() -> holder.getConnection().getStatements(subject, predicate, object)); } public void remove(Resource subject, IRI predicate, Value object) { - holder.getConnection().remove(subject, predicate, object); + getNamedGraphIri().ifPresentOrElse( + namedGraphIri -> holder.getConnection().remove(subject, predicate, object, namedGraphIri), + () -> holder.getConnection().remove(subject, predicate, object) + ); } public void commit() { @@ -45,11 +48,18 @@ public void rollback() { connection.close(); } - public void executeUpdate(String query) { - holder.getConnection().prepareUpdate(query).execute(); + public void close() { + holder.getConnection().close(); } public void shutdown() { holder.shutdown(); } + + private Optional getNamedGraphIri() { + return Optional + .ofNullable(namedGraph) + .filter(graph -> !graph.isEmpty()) + .map(SimpleValueFactory.getInstance()::createIRI); + } } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java index 2410e8fb2..71be4e28f 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java @@ -2,9 +2,10 @@ import org.apache.jena.riot.RDFParser; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.util.Models; import org.eclipse.rdf4j.repository.config.RepositoryConfig; import org.eclipse.rdf4j.repository.manager.LocalRepositoryManager; import org.eclipse.rdf4j.repository.manager.RepositoryManager; @@ -15,37 +16,37 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; class MaterialiserIT { - private static final String LOCAL_SERVER_URL = "http://localhost:8080/rdf4j-server"; private static final String LOCAL_REPOSITORY_ID = "test"; private static Materialiser materialiser; + private RepositoryManager subject; @TempDir File dataDir; @BeforeEach public void setUp() { - materialiser = new Materialiser(LOCAL_SERVER_URL, LOCAL_REPOSITORY_ID, ""); - final RepositoryManager subject = new LocalRepositoryManager(dataDir); + subject = new LocalRepositoryManager(dataDir); subject.init(); subject.addRepositoryConfig( new RepositoryConfig(LOCAL_REPOSITORY_ID, new SailRepositoryConfig(new MemoryStoreConfig(true)))); - - materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, ""); } @AfterEach @@ -53,27 +54,31 @@ public void tearDown() { materialiser.shutdown(); } - @Test - void when_DeleteEntities_Then_EntitiesRemovedFromStore() throws Exception { + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void when_DeleteEntities_Then_EntitiesRemovedFromStore(String namedGraph) throws Exception { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + + final int statementOfModel2And3Count = 9; final List testFiles = IntStream.range(1, 4).mapToObj("src/test/resources/people/%d.nq"::formatted).toList(); - populateAndCheckRepository(testFiles); - final Model model = Rio.parse(new FileInputStream("src/test/resources/people/1.nq"), "", RDFFormat.NQUADS); - Set entityIds = Stream.of("http://somewhere/BeckySmith/", "http://somewhere/SarahJones/") - .map(iri -> SimpleValueFactory.getInstance().createIRI(iri)) - .collect(Collectors.toSet()); + populateAndCheckRepository(testFiles, namedGraph); + final Model modelToDelete = Rio.parse(new FileInputStream("src/test/resources/people/1.nq"), "", RDFFormat.NQUADS); - materialiser.deleteEntities(entityIds); + materialiser.deleteEntity(modelToDelete); List statements = materialiser.getMaterialiserConnection() .getStatements(null, null, null).stream().toList(); - assertThat(statements).hasSize(4); + assertThat(statements).hasSize(statementOfModel2And3Count); } - @Test - void when_UpdateEntities_Then_OldTriplesRemoved() throws Exception { + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void when_UpdateEntities_Then_OldTriplesRemoved(String namedGraph) throws Exception { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + final List testFiles = IntStream.range(1, 6).mapToObj("src/test/resources/people/%d.nq"::formatted).toList(); - populateAndCheckRepository(testFiles); + populateAndCheckRepository(testFiles, namedGraph); List models = List.of(RDFParser.source("people/5-updated.nq").toModel()); materialiser.process(models); @@ -87,7 +92,66 @@ void when_UpdateEntities_Then_OldTriplesRemoved() throws Exception { .noneMatch(statement -> statement.getObject().stringValue().equals("Taylor")); } - void populateAndCheckRepository(List files) throws IOException { + @Test + void given_TwoDifferentNamedGraphs_when_processModelsWithSameSubjects_then_ExpectNoChangesInOtherNamedGraph() { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, "http://example.org/named-graph-1"); + final Materialiser secondMaterialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, "http://example.org/named-graph-2"); + + final org.apache.jena.rdf.model.Model firstJenaModel = RDFParser.source("movies/1.ttl").toModel(); + final org.apache.jena.rdf.model.Model secondJenaModel = RDFParser.source("movies/2.ttl").toModel(); + materialiser.process(List.of(firstJenaModel)); + secondMaterialiser.process(List.of(secondJenaModel)); + + Model firstModel = new LinkedHashModel(); + Model secondModel = new LinkedHashModel(); + + materialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(firstModel::add); + + + secondMaterialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(secondModel::add); + + assertThat(firstModel) + .hasSize((int) firstJenaModel.size()); + + assertThat(secondModel) + .hasSize((int) secondJenaModel.size()); + + materialiser.closeConnection(); + secondMaterialiser.closeConnection(); + } + + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void given_ComplexModelToUpdate_test_Process(String namedGraph) throws IOException { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + + final org.apache.jena.rdf.model.Model originalModel = RDFParser.source("movies/1.ttl").toModel(); + materialiser.process(List.of(originalModel)); + + final org.apache.jena.rdf.model.Model updatedJenaModel = RDFParser.source("movies/2.ttl").toModel(); + final Model updatedRdfModel = Rio.parse(new FileInputStream("src/test/resources/movies/2.ttl"), "", RDFFormat.TURTLE); + materialiser.process(List.of(updatedJenaModel)); + + + Model updatedDbModel = new LinkedHashModel(); + materialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(updatedDbModel::add); + + assertThat(Models.isomorphic(updatedDbModel, updatedRdfModel)).isTrue(); + } + + void populateAndCheckRepository(List files, String namedGraph) throws IOException { List models = new ArrayList<>(); for (String testFile : files) { var model = Rio.parse(new FileInputStream(testFile), "", RDFFormat.NQUADS); @@ -96,8 +160,24 @@ void populateAndCheckRepository(List files) throws IOException { } List statements = materialiser.getMaterialiserConnection().getStatements(null, null, null).stream().toList(); + if (!namedGraph.isEmpty()) { + statements = statements.stream() + .map(MaterialiserIT::deleteContextFromStatement) + .toList(); + } assertThat(models).allMatch(statements::containsAll); } + private static Statement deleteContextFromStatement(Statement statement) { + return SimpleValueFactory.getInstance().createStatement(statement.getSubject(), statement.getPredicate(), statement.getObject()); + } + + static class NamedGraphProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext extensionContext) { + return Stream.of("", "http://example.org/my-named-graph").map(Arguments::of); + } + } + } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java index b2f758100..f056a4d18 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserTest.java @@ -11,7 +11,9 @@ import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.RepositoryException; +import org.eclipse.rdf4j.repository.RepositoryResult; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +import org.eclipse.rdf4j.repository.sparql.federation.CollectionIteration; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.Rio; import org.junit.jupiter.api.AfterEach; @@ -41,8 +43,6 @@ class MaterialiserTest { private RepositoryManager repositoryManager; @Mock private Repository repository; - @Mock - private Update update; private Materialiser materialiser; @BeforeEach @@ -50,7 +50,6 @@ void setUp() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); materialiser = new Materialiser(repositoryManager, REPOSITORY_ID, ""); - when(connection.prepareUpdate(anyString())).thenReturn(update); } @AfterEach @@ -63,24 +62,33 @@ void when_DeleteEntities_Then_EntitiesRemovedFromStore() throws Exception { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); Set entityIds = Stream - .of("http://somewhere/SarahJones/", "http://somewhere/MattJones/") + .of("http://somewhere/DickJones/", "http://somewhere/SarahJones/") .map(stringUri -> SimpleValueFactory.getInstance().createIRI(stringUri)) .collect(Collectors.toSet()); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); - materialiser.deleteEntities(entityIds); + Model modelToDelete = Rio.parse(new FileInputStream(CHANGED_FILE), "", RDFFormat.NQUADS); - verify(update, times(2)).execute(); + materialiser.deleteEntity(modelToDelete); + entityIds.forEach(subjectIri -> { + verify(connection).getStatements(subjectIri, null, null); + verify(connection).remove(subjectIri, null, null); + }); } @Test void given_RepositoryContainingData_when_UpdateSingleModel_Then_OldTriplesRemoved() throws Exception { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); Model changedModel = Rio.parse(new FileInputStream(CHANGED_FILE), "", RDFFormat.NQUADS); materialiser.process(List.of(RDFParser.source(CHANGED_FILE).toModel())); - verify(update, times(2)).execute(); + verify(connection).remove(SimpleValueFactory.getInstance().createIRI("http://somewhere/DickJones/"), null, null); + verify(connection).remove(SimpleValueFactory.getInstance().createIRI("http://somewhere/SarahJones/"), null, null); verify(connection).add(changedModel); verify(connection).commit(); } @@ -89,12 +97,14 @@ void given_RepositoryContainingData_when_UpdateSingleModel_Then_OldTriplesRemove void given_ValidListOfMembers_when_ProcessList_then_CommitToRepository() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); List models = readTenModelsFromFile().toList(); materialiser.process(models); - verify(update, times(10)).execute(); + verify(connection, times(10)).remove(any(Resource.class), isNull(), isNull()); verify(connection, times(10)).add(any(Model.class)); verify(connection).commit(); } @@ -103,6 +113,8 @@ void given_ValidListOfMembers_when_ProcessList_then_CommitToRepository() { void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); doThrow(RepositoryException.class).when(connection).commit(); List models = readTenModelsFromFile().toList(); @@ -111,7 +123,6 @@ void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() .isInstanceOf(MaterialisationFailedException.class) .hasCauseInstanceOf(RepositoryException.class); - verify(update, times(10)).execute(); verify(connection, times(10)).add(any(Model.class)); verify(connection).commit(); verify(connection).rollback(); @@ -121,6 +132,8 @@ void given_ValidList_when_ProcessList_and_CommitFails_then_RollbackConnection() void given_ListOfMembers_when_ProcessList_and_AddModelToConnectionFails_then_StopAdditionAndRollback() { when(repositoryManager.getRepository(REPOSITORY_ID)).thenReturn(repository); when(repository.getConnection()).thenReturn(connection); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))); doNothing().doNothing().doThrow(RepositoryException.class).when(connection).add(any(Model.class)); List models = readTenModelsFromFile().toList(); @@ -136,11 +149,11 @@ void given_ListOfMembers_when_ProcessList_and_AddModelToConnectionFails_then_Sto @Test void given_ListOfMembers_when_ProcessList_and_GetStatementsFails_then_StopAdditionAndRollback() { - doNothing() - .doNothing() - .doNothing() - .doThrow(RepositoryException.class) - .when(update).execute(); + when(connection.getStatements(any(), isNull(), isNull())) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenReturn(new RepositoryResult<>(new CollectionIteration<>(Set.of()))) + .thenThrow(RepositoryException.class); List models = readTenModelsFromFile().toList(); @@ -148,11 +161,10 @@ void given_ListOfMembers_when_ProcessList_and_GetStatementsFails_then_StopAdditi .isInstanceOf(MaterialisationFailedException.class) .hasCauseInstanceOf(RepositoryException.class); - verify(update, times(4)).execute(); - verify(connection, times(3)).add(any(Model.class)); + verify(connection, times(4)).getStatements(any(), isNull(), isNull()); + verify(connection, times(3)).remove(any(Resource.class), isNull(), isNull()); verify(connection, times(3)).add(any(Model.class)); verify(connection, never()).commit(); verify(connection).rollback(); - } private Stream readTenModelsFromFile() { diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl new file mode 100644 index 000000000..3cee8d5d9 --- /dev/null +++ b/ldi-core/repository-materialiser/src/test/resources/movies/1.ttl @@ -0,0 +1,40 @@ +@prefix schema: . +@prefix rdf: . +@prefix skos: . +@prefix xsd: . + + a schema:Movie ; + schema:name "Dune" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel."@en, "Een epische verfilming van de sci-fi novel van Frank Herbert."@nl ; + schema:datePublished "2021-09-15"^^xsd:date ; + schema:nominated true ; + schema:director [ + a schema:Person ; + schema:name "Denis Villeneuve" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Director" ; + skos:broader [ + a skos:Concept ; + skos:prefLabel "Person" + ] + ] + ] ; + schema:actor [ + a schema:Person ; + schema:name "Timothée Chalamet" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ], [ + a schema:Person ; + schema:name "Rebecca Ferguson" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ] ; + schema:genre "Science Fiction", "Adventure" ; + schema:duration "PT155M"^^xsd:duration ; + schema:language "English" . \ No newline at end of file diff --git a/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl new file mode 100644 index 000000000..1c867b99b --- /dev/null +++ b/ldi-core/repository-materialiser/src/test/resources/movies/2.ttl @@ -0,0 +1,36 @@ +@prefix schema: . +@prefix rdf: . +@prefix skos: . +@prefix xsd: . + + a schema:Movie ; + schema:name "Dune: Part Two" ; + schema:description "An epic adaptation of Frank Herbert's sci-fi novel."@en, "Een epische verfilming van de sci-fi novel van Frank Herbert."@nl ; + schema:datePublished "2024-02-28"^^xsd:date ; + schema:nominated false ; + schema:director [ + a schema:Person ; + schema:name "Denis Villeneuve" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Director" + ] + ] ; + schema:actor [ + a schema:Person ; + schema:name "Timothée Chalamet" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ], [ + a schema:Person ; + schema:name "Rebecca Ferguson" ; + schema:role [ + a skos:Concept ; + skos:prefLabel "Actor" + ] + ] ; + schema:genre "Science Fiction", "Adventure" ; + schema:duration "PT167M"^^xsd:duration ; + schema:language "English" . \ No newline at end of file diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index d7a0b9f0f..f79dae4a4 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index 530b461b2..60d2b3f76 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -25,17 +25,20 @@ public class ClientCredentialsConfig implements RequestExecutorSupplier { private final String secret; private final String tokenEndpoint; private final String scope; + private final boolean enableRedirectHandling; public ClientCredentialsConfig(Collection
headers, String clientId, String secret, String tokenEndpoint, - String scope) { + String scope, + boolean enableRedirectHandling) { this.headers = headers; this.clientId = notNull(clientId); this.secret = notNull(secret); this.tokenEndpoint = notNull(tokenEndpoint); this.scope = scope; + this.enableRedirectHandling = enableRedirectHandling; } public RequestExecutor createRequestExecutor() { @@ -43,7 +46,8 @@ public RequestExecutor createRequestExecutor() { } private OAuth20Service createService() { - final RequestConfig clientConfig = RequestConfig.custom().setRedirectsEnabled(false).build(); + final RequestConfig.Builder configBuilder = RequestConfig.custom(); + final RequestConfig clientConfig = configBuilder.setRedirectsEnabled(enableRedirectHandling).build(); final ApacheHttpClient apacheHttpClient = new ApacheHttpClient( HttpAsyncClientBuilder.create().setDefaultRequestConfig(clientConfig).setDefaultHeaders(headers).build()); final DefaultApi20 authorizationApi = createAuthorizationApi(tokenEndpoint); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java index bdf52ad22..e6dd85597 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenService.java @@ -16,11 +16,17 @@ public class MemoryTokenService implements TokenService { private EdcToken token; - public MemoryTokenService(TransferService transferService) { + private final MemoryTokenServiceLifecycle lifecycle; + + public MemoryTokenService(TransferService transferService, MemoryTokenServiceLifecycle lifecycle) { this.transferService = transferService; - } + this.lifecycle = lifecycle; + } + @Override public RequestHeader waitForTokenHeader() { + lifecycle.checkPipelineState(); + if (token != null) { return token.getTokenHeader(); } @@ -45,4 +51,19 @@ public void updateToken(String token) { this.token = EdcToken.fromJsonString(token); } + @Override + public void pause() { + lifecycle.pause(); + } + + @Override + public void resume() { + lifecycle.resume(); + } + + @Override + public void shutdown() { + lifecycle.shutdown(); + } + } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java new file mode 100644 index 000000000..e5b100a29 --- /dev/null +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycle.java @@ -0,0 +1,51 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MemoryTokenServiceLifecycle { + + private final Logger log = LoggerFactory.getLogger(MemoryTokenServiceLifecycle.class); + + private boolean terminated = false; + private boolean paused = false; + + void checkPipelineState() { + checkPause(); + checkTerminated(); + } + + private synchronized void checkPause() { + while (paused) { + try { + this.wait(); + } catch (InterruptedException e) { + log.error("Thread interrupted: {}", e.getMessage()); + Thread.currentThread().interrupt(); + } + } + } + + private void checkTerminated() { + if (terminated) { + throw new TerminatedException(); + } + } + + void pause() { + this.paused = true; + } + + synchronized void resume() { + this.paused = false; + this.notifyAll(); + } + + void shutdown() { + this.terminated = true; + } + + public static class TerminatedException extends RuntimeException { + } + +} diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java index 58db9d9b7..db7a073db 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java @@ -10,4 +10,9 @@ public interface TokenService { void updateToken(String token); + void pause(); + + void resume(); + + void shutdown(); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java index d3bd09a6d..eebba062b 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java @@ -10,14 +10,20 @@ public class DefaultConfig implements RequestExecutorSupplier { private final Collection
headers; + private final boolean enableRedirectHandling; - public DefaultConfig(Collection
headers) { + public DefaultConfig(Collection
headers, boolean enableRedirectHandling) { this.headers = headers; - } + this.enableRedirectHandling = enableRedirectHandling; + } public RequestExecutor createRequestExecutor() { - return new DefaultRequestExecutor( - HttpClientBuilder.create().setDefaultHeaders(headers).disableRedirectHandling().build()); + final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + if (!enableRedirectHandling) { + httpClientBuilder.disableRedirectHandling(); + } + + return new DefaultRequestExecutor(httpClientBuilder.setDefaultHeaders(headers).build()); } } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java index 15676fd50..2189c1d66 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/services/RequestExecutorFactory.java @@ -13,8 +13,18 @@ public class RequestExecutorFactory { + private final boolean enableRedirectHandling; + + public RequestExecutorFactory() { + this(true); + } + + public RequestExecutorFactory(boolean enableRedirectHandling) { + this.enableRedirectHandling = enableRedirectHandling; + } + public RequestExecutor createNoAuthExecutor(Collection
headers) { - return new DefaultConfig(headers).createRequestExecutor(); + return new DefaultConfig(headers, enableRedirectHandling).createRequestExecutor(); } public RequestExecutor createNoAuthExecutor() { @@ -26,7 +36,8 @@ public RequestExecutor createClientCredentialsExecutor(Collection
header String secret, String tokenEndpoint, String scope) { - return new ClientCredentialsConfig(headers, clientId, secret, tokenEndpoint, scope).createRequestExecutor(); + var config = new ClientCredentialsConfig(headers, clientId, secret, tokenEndpoint, scope, enableRedirectHandling); + return config.createRequestExecutor(); } public RequestExecutor createClientCredentialsExecutor(String clientId, diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java index 3957bd4c0..a727af65e 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java @@ -28,7 +28,7 @@ public class RequestExecutorSteps { private LocalDateTime start; - private final RequestExecutorFactory factory = new RequestExecutorFactory(); + private final RequestExecutorFactory factory = new RequestExecutorFactory(false); private RequestExecutor requestExecutor; private Response response; private Request request; diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java new file mode 100644 index 000000000..12fac48fa --- /dev/null +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceLifecycleTest.java @@ -0,0 +1,60 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.SequencedCollection; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class MemoryTokenServiceLifecycleTest { + + private MemoryTokenServiceLifecycle serviceLifecycle; + + @BeforeEach + void setUp() { + serviceLifecycle = new MemoryTokenServiceLifecycle(); + } + + @Test + void shouldPauseAndResume() throws InterruptedException { + final SequencedCollection tracker = new ArrayList<>(); + + Thread testThread = new Thread(() -> { + serviceLifecycle.pause(); + tracker.addLast("The lifecycle is paused. Let's check the state!"); + serviceLifecycle.checkPipelineState(); // This should block until resumed + tracker.addLast("Finally, we have resumed!"); + }); + + testThread.start(); + await().atLeast(100, TimeUnit.MILLISECONDS); // Ensures the thread has time to pause + tracker.addLast("The thread with the lifecycle should now be waiting until someone resumes the lifecycle."); + await().atLeast(100, TimeUnit.MILLISECONDS); + tracker.addLast("After a minor nap, the lifecycle should still be waiting.."); + + serviceLifecycle.resume(); + testThread.join(); + + assertThat(tracker.getLast()).isEqualTo("Finally, we have resumed!"); + assertDoesNotThrow(() -> serviceLifecycle.checkPipelineState()); + } + + @Test + void whenLifecycleIsRunning_shouldNotThrowExceptionsOrHang() { + assertDoesNotThrow(() -> serviceLifecycle.checkPipelineState()); + } + + @Test + void whenLifecycleIsShutDown_ThenTerminatedExceptionIsThrown() { + serviceLifecycle.shutdown(); + assertThatThrownBy(() -> serviceLifecycle.checkPipelineState()) + .isInstanceOf(MemoryTokenServiceLifecycle.TerminatedException.class); + } + +} \ No newline at end of file diff --git a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java index e2f192e9a..55c3c96e7 100644 --- a/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java +++ b/ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/MemoryTokenServiceTest.java @@ -16,6 +16,9 @@ @ExtendWith(MockitoExtension.class) class MemoryTokenServiceTest { + @Mock + private MemoryTokenServiceLifecycle memoryTokenServiceLifecycle; + @Mock private TransferService transferService; diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 272287ae9..f843392fd 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index e445ef069..daa3e5315 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 9235a91be..58f59287f 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 137f3ebb8..00afe2785 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 168cb466a..da524ef79 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java index 815d0fafd..1eb04cc3c 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java @@ -18,7 +18,7 @@ public class LdesDiscovererExecutor implements CommandLineRunner { public LdesDiscovererExecutor(LdesDiscovererConfig config) { this.config = config; - final RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + final RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); ldesStructureDiscoverer = new LdesStructureDiscoverer(config.getUrl(), config.getSourceFormatAsLang(), requestExecutor); } @@ -26,9 +26,13 @@ public LdesDiscovererExecutor(LdesDiscovererConfig config) { public void run(String... args) { log.info("Running LDESDiscoverer for url {}", config.getUrl()); - final LdesStructure ldesStructure = ldesStructureDiscoverer.discoverLdesStructure(); + try { + final LdesStructure ldesStructure = ldesStructureDiscoverer.discoverLdesStructure(); - log.atInfo().log(ldesStructure.toString()); + log.atInfo().log(ldesStructure.toString()); + } catch (Exception e) { + log.atError().log(e.getMessage()); + } } } \ No newline at end of file diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java index ad3c2f14d..a9aface78 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java @@ -8,8 +8,7 @@ @Configuration @ConfigurationProperties public class LdesDiscovererConfig { - public static final String DEFAULT_SOURCE_FORMAT = "application/n-quads"; - public static final String DEFAULT_OUTPUT_FORMAT = "text/turtle"; + public static final String DEFAULT_RDF_FORMAT = "text/turtle"; private String url; private String sourceFormat; private String outputFormat; @@ -26,7 +25,7 @@ public void setUrl(String url) { } public String getSourceFormat() { - return sourceFormat == null ? DEFAULT_SOURCE_FORMAT : sourceFormat; + return sourceFormat == null ? DEFAULT_RDF_FORMAT : sourceFormat; } public Lang getSourceFormatAsLang() { @@ -38,7 +37,7 @@ public void setSourceFormat(String sourceFormat) { } public String getOutputFormat() { - return outputFormat == null ? DEFAULT_OUTPUT_FORMAT : outputFormat; + return outputFormat == null ? DEFAULT_RDF_FORMAT : outputFormat; } public Lang getOutputFormatAsLang() { diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java index e71945091..e94ffce32 100644 --- a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfigTest.java @@ -18,7 +18,7 @@ void setUp() { @Test void given_EmptyConfig_when_GetSourceFormatAsLang_then_ReturnNQuads() { - final Lang defaultSourceLang = Lang.NQUADS; + final Lang defaultSourceLang = Lang.TURTLE; final Lang actualSourceLang = config.getSourceFormatAsLang(); diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index 181a6db33..148ffb99f 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi.Dockerfile b/ldi-nifi.Dockerfile index 2536325cd..7ee77cd6b 100644 --- a/ldi-nifi.Dockerfile +++ b/ldi-nifi.Dockerfile @@ -3,7 +3,7 @@ ARG NIFI_DOCKER_IMAGE_VERSION # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 4094129aa..1d6890948 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 445063330..667b56b7e 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index fb9bbc796..c7bd29879 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index c2ce1b1db..a6f8cfcfa 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index 3f783a002..46f7626d2 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index 3bd415d67..68fac0e64 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 429f541d1..91b3e3f23 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 @@ -34,7 +34,7 @@ net.bytebuddy byte-buddy-dep - ${byte-buddy-dep.version} + ${byte-buddy.version} org.testcontainers diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index 425477f7b..684e1d0cf 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -55,7 +56,7 @@ public class LdesClientProcessor extends AbstractProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(LdesClientProcessor.class); private MemberSupplier memberSupplier; private LdesProperties ldesProperties; - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); private boolean hasLdesEnded; private boolean keepState; diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index 90c8ff1e2..37aacee46 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 3e3ad1dbc..60da9d89e 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 4bdd37a1f..61339c851 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java index bed1d8c74..e2599226d 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RDF4JRepositoryMaterialisationProcessor.java @@ -89,7 +89,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro session.transfer(flowFiles, SUCCESS); } catch (Exception e) { - getLogger().error("Error sending model to repository", e.getMessage()); + getLogger().error("Error sending model to repository: {}", e.getMessage()); session.transfer(flowFiles, FAILURE); } } diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index 9a1ca0181..f7e0651f3 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index 175958140..d33a379ca 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index b9ebad52d..9a7c496fb 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 838ccdd39..ae6ef7502 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java index d887afde7..78e079aa7 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.*; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; @Component @@ -54,30 +55,19 @@ public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { currentSavedPipeline.updateStatus(statusEvent.status(), statusEvent.statusChangeSource()); } - @EventListener - public void handlePipelineDeleted(PipelineDeletedEvent deletedEvent) { - this.savedPipelines.remove(deletedEvent.pipelineId()); - } - @EventListener public void handlePipelineCreated(InputCreatedEvent event) { - savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), RUNNING)); + savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), event.ldioInput().getStatus())); } public PipelineStatus resumeHaltedPipeline(String pipelineId) { - PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); + var pipelineStatus = getPipelineStatus(pipelineId); return switch (pipelineStatus) { - case HALTED -> { - savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUMING); - yield RESUMING; - } + case HALTED -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUME); case INIT -> INIT; - case STARTING -> STARTING; case RUNNING -> RUNNING; - case RESUMING -> RESUMING; case STOPPED -> STOPPED; - case STOPPING -> STOPPING; }; } @@ -85,23 +75,18 @@ public PipelineStatus haltRunningPipeline(String pipelineId) { PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); return switch (pipelineStatus) { - case RUNNING, RESUMING -> { - savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALTED); - yield HALTED; - } + case RUNNING -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALT); case INIT -> INIT; - case STARTING -> STARTING; case HALTED -> HALTED; case STOPPED -> STOPPED; - case STOPPING -> STOPPING; }; } public PipelineStatus stopPipeline(String pipelineId) { - LdioInput input = savedPipelines.get(pipelineId).getLdioInput(); - input.updateStatus(STOPPING); + PipelineStatus newPipelineStatus = savedPipelines.get(pipelineId).getLdioInput().updateStatus(STOP); + this.savedPipelines.remove(pipelineId); eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineId)); - return STOPPED; + return newPipelineStatus; } public Map getPipelineStatusOverview() { diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java index 95748c607..a882fa3be 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java @@ -5,14 +5,14 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import org.springframework.context.ApplicationEventPublisher; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; public class DummyIn extends LdioInput { private int counter = 0; public DummyIn(ComponentExecutor executor, LdiAdapter adapter, ApplicationEventPublisher applicationEventPublisher) { super("DummyIn", "test", executor, adapter, null, applicationEventPublisher); - this.updateStatus(STARTING); + this.updateStatus(START); } public void sendData() { diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java index 8e86f3f19..03189b5fc 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.context.ApplicationEventPublisher; @@ -24,10 +25,11 @@ void setup() { @Test void when_StoppingPipeline_Then_MethodsAreCalled() { + when(input.updateStatus(any())).thenReturn(PipelineStatus.STOPPED); PipelineStatus result = pipelineStatusService.stopPipeline(pipelineName); assertEquals(PipelineStatus.STOPPED, result); - verify(input).updateStatus(PipelineStatus.STOPPING); + verify(input).updateStatus(PipelineStatusTrigger.STOP); } } diff --git a/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature b/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature index c8e935d67..9293c10bf 100644 --- a/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature +++ b/ldi-orchestrator/ldio-application/src/test/resources/features/management.feature @@ -9,8 +9,8 @@ Feature: Management of pipelines in LDIO And I expect pipelines Examples: | file | type | statusCode | pipelineCount | - | "valid" | "json" | 200 | 1 | - | "valid" | "yml" | 200 | 1 | + | "valid" | "json" | 201 | 1 | + | "valid" | "yml" | 201 | 1 | | "invalid" | "json" | 400 | 0 | | "invalid" | "yml" | 400 | 0 | | "missing-adapter" | "yml" | 400 | 0 | @@ -23,19 +23,19 @@ Feature: Management of pipelines in LDIO And I expect pipelines Examples: | firstfile | firstStatusCode | secondFile | secondStatusCode | type | pipelineCount | - | "valid" | 200 | "valid" | 400 | "json" | 1 | - | "valid" | 200 | "valid" | 400 | "yml" | 1 | - | "valid" | 200 | "valid-2" | 200 | "json" | 2 | - | "valid" | 200 | "valid-2" | 200 | "yml" | 2 | - | "valid" | 200 | "invalid" | 400 | "json" | 1 | - | "valid" | 200 | "invalid" | 400 | "yml" | 1 | + | "valid" | 201 | "valid" | 400 | "json" | 1 | + | "valid" | 201 | "valid" | 400 | "yml" | 1 | + | "valid" | 201 | "valid-2" | 201 | "json" | 2 | + | "valid" | 201 | "valid-2" | 201 | "yml" | 2 | + | "valid" | 201 | "invalid" | 400 | "json" | 1 | + | "valid" | 201 | "invalid" | 400 | "yml" | 1 | | "invalid" | 400 | "invalid" | 400 | "json" | 0 | | "invalid" | 400 | "invalid" | 400 | "yml" | 0 | @deletion Scenario: Deleting a pipeline When I start LDIO - And I post a "valid" "yml" pipeline with a 200 response + And I post a "valid" "yml" pipeline with a 201 response And I delete the "valid-pipeline" pipeline with a 202 response And I expect 0 pipelines diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 24684abf7..c3de76ca7 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index c8cb49591..1bb205529 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -6,6 +6,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import io.micrometer.core.instrument.Metrics; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationRegistry; @@ -16,7 +17,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.MANUAL; /** @@ -94,41 +95,36 @@ protected void processModel(Model model) { public abstract void shutdown(); - public void updateStatus(PipelineStatus statusEvent) { - switch (statusEvent) { - case STARTING -> { - this.pipelineStatus = RUNNING; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, AUTO)); - } - case RESUMING -> { + public PipelineStatus updateStatus(PipelineStatusTrigger trigger) { + switch (trigger) { + case START -> this.pipelineStatus = RUNNING; + case RESUME -> { this.resume(); this.pipelineStatus = RUNNING; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, RUNNING, MANUAL)); } - case HALTED -> { + case HALT -> { if (this.pipelineStatus != INIT) { this.pause(); this.pipelineStatus = HALTED; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, HALTED, MANUAL)); } } - case STOPPING -> { - this.pipelineStatus = STOPPED; - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, STOPPED, MANUAL)); - } - default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, statusEvent); + case STOP -> this.pipelineStatus = STOPPED; + default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, pipelineStatus); } + + applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, this.pipelineStatus, MANUAL)); + return this.pipelineStatus; } protected abstract void resume(); protected abstract void pause(); - public void starting() { - this.pipelineStatus = STARTING; + public void start() { + updateStatus(START); } - public boolean isHalted() { - return pipelineStatus.equals(HALTED) || pipelineStatus.equals(STOPPED); + public PipelineStatus getStatus() { + return this.pipelineStatus; } } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java index 20228b5fa..c5c1cb28a 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatus.java @@ -1,5 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; public enum PipelineStatus { - INIT, STARTING, RESUMING, RUNNING, HALTED, STOPPING, STOPPED + INIT, RUNNING, HALTED, STOPPED } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java new file mode 100644 index 000000000..9f792486d --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineStatusTrigger.java @@ -0,0 +1,7 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; + +public enum PipelineStatusTrigger { + + START, HALT, RESUME, STOP + +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index cc5bbea3b..04bf8506a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java index c8aa08027..1ce02e87b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java @@ -45,7 +45,7 @@ public LdioAmqpIn(String pipelineName, ComponentExecutor executor, LdiAdapter ad ldioAmqpInRegistrator = jmsInRegistrator; listenerId = endpoint.getId(); jmsInRegistrator.registerListener(jmsConfig, endpoint); - this.starting(); + this.start(); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java index 0da71b9e3..05c2e3d02 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java @@ -29,8 +29,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.HALT; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.RESUME; import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; import static org.apache.jena.riot.RDFLanguages.nameToLang; import static org.awaitility.Awaitility.await; @@ -103,6 +103,7 @@ public void iCreateDefaultConfigForLdioJmsInWithContentType(String contentType) public void theListenerWillWaitForTheMessage(int i) { await().until(() -> adapterResult.size() == i); } + @Then("Wait for a grace period") public void waitForGracePeriod() throws InterruptedException { Awaitility.waitAtMost(Duration.of(500, ChronoUnit.MILLIS)); @@ -117,12 +118,14 @@ public void theResultValueWillContainTheModel() { @When("I pause the pipeline") public void pausePipeline() { - ldioInput.updateStatus(HALTED); + ldioInput.updateStatus(HALT); } + @When("I unpause the pipeline") public void unPausePipeline() { - ldioInput.updateStatus(RESUMING); + ldioInput.updateStatus(RESUME); } + @And("The result value will not contain the model") public void theResultValueWillNotContainTheModel() { assertEquals(0, adapterResult.size()); @@ -137,4 +140,5 @@ public void theComponentExecutorWillHaveBeenCalled(int interactions) { public Boolean booleanValue(String value) { return Boolean.valueOf(value); } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index ac80b568b..aef2490e5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index 03c8b0519..6ce6e259d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -21,7 +21,7 @@ public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, Observ super(NAME, pipelineName, executor, null, observationRegistry, applicationEventPublisher); this.archiveFileCrawler = crawler; this.sourceFormat = source; - starting(); + start(); log.info("Starting with crawling the archive."); crawlArchive(); log.info("Finished crawling the archive."); diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 7ce87bb6a..62c774e50 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 0adaac49c..8b858844a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 21b8b0578..50d954b7e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index 797107211..f7745cf8c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index c38d1491c..35b8af2ed 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index b6f4b2c66..5d7d3f716 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java index 7cea91e35..5555f8889 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java @@ -91,6 +91,8 @@ private void executeRequest(Request request) { String content = response.getBodyAsString().orElseThrow(); processInput(content, contentType); } else { + log.error("Failed to execute request {} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); + response.getBodyAsString().ifPresent(log::error); throw new UnsuccesfulPollingException(response.getHttpStatus(), request.getUrl()); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java index 2e9b414ca..e8836a576 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java @@ -53,7 +53,7 @@ public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor execu httpInputPoller.schedulePoller(getPollingInterval(properties)); - httpInputPoller.starting(); + httpInputPoller.start(); return httpInputPoller; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 042ef0d61..8592214eb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index ce99381ff..7e77bd331 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -44,7 +44,7 @@ public LdioInput configure(LdiAdapter adapter, LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); eventPublisher.publishEvent(new HttpInPipelineCreatedEvent(pipelineName, ldioHttpIn)); - ldioHttpIn.starting(); + ldioHttpIn.start(); return ldioHttpIn; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java index ce82f5cbe..81c34803b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputTest.java @@ -17,8 +17,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInProcess.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.HALTED; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.RESUMING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.HALT; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.RESUME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -50,7 +50,7 @@ void setup() { void testHttpEndpoint() throws Exception { String content = "_:b0 \"Jane Doe\" ."; String contentType = "application/n-quads"; - input.updateStatus(RESUMING); + input.updateStatus(RESUME); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); @@ -60,13 +60,13 @@ void testHttpEndpoint() throws Exception { void when_PipelineIsHalted_Then_MessageIsNotProcessed() throws Exception { String content = "_:b0 \"Jane Doe\" ."; String contentType = "application/n-quads"; - input.updateStatus(HALTED); + input.updateStatus(HALT); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().is(503)); verifyNoInteractions(adapter); - input.updateStatus(RESUMING); + input.updateStatus(RESUME); mockMvc.perform(post("/%s".formatted(endpoint)).content(content).contentType(contentType)).andExpect(status().isAccepted()); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 03f2e453f..31dac60e2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index d9954ac86..557c3c519 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 96f5ab1f5..6fa14f59b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index 7bfae23bd..17f8c4e82 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -51,7 +51,7 @@ public LdioKafkaIn(String pipelineName, ComponentExecutor executor, LdiAdapter a containerProps.setMessageListener(listener); this.container = new KafkaMessageListenerContainer<>(consumerFactory, containerProps); container.start(); - this.starting(); + this.start(); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java index 55583cd3c..252d662a8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/KafkaInIntegrationTestSteps.java @@ -4,7 +4,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -110,6 +110,7 @@ public void iSendTheModelToTheUsingTheKafkaProducer() { public void theListenerWillWaitForTheMessage() { await().until(() -> adapterResult.size() == 1); } + @Then("Wait for a grace period") public void theListenerWillWaitForPeriod() throws InterruptedException { Awaitility.waitAtMost(1500, TimeUnit.MILLISECONDS); @@ -131,14 +132,15 @@ public void theResultValueWillContainTheModel() { public void theComponentExecutorWillHaveBeenCalled(int i) { assertEquals(i, componentExecutorResult.size()); } + @When("I pause the pipeline") public void pauseInput() { - kafkaIn.updateStatus(PipelineStatus.HALTED); + kafkaIn.updateStatus(PipelineStatusTrigger.HALT); } + @When("I unpause the pipeline") public void unPauseInput() { - kafkaIn.updateStatus(PipelineStatus.RESUMING); + kafkaIn.updateStatus(PipelineStatusTrigger.RESUME); } -} - +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index b4dba2581..40cfe4b93 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index d89ccbaeb..ba061866b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TransferService; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.STARTING; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; public class LdioLdesClientConnectorApi { private final TransferService transferService; @@ -22,9 +22,22 @@ public void handleToken(String token) { } public String handleTransfer(String transfer) { - String transer = transferService.startTransfer(transfer).getBodyAsString() + String response = transferService.startTransfer(transfer).getBodyAsString() .orElse(""); - ldesClient.updateStatus(STARTING); - return transer; + ldesClient.updateStatus(START); + return response; } + + public void shutdown() { + tokenService.shutdown(); + } + + public void resume() { + tokenService.resume(); + } + + public void pause() { + tokenService.pause(); + } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java index 39e202431..6f6428167 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApiController.java @@ -1,6 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; +import be.vlaanderen.informatievlaanderen.ldes.ldio.collection.LdioLdesClientConnectorApiCollection; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import org.springframework.context.event.EventListener; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -8,18 +11,17 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Optional.ofNullable; - @RestController public class LdioLdesClientConnectorApiController { - private final Map clientConnectorApis = new HashMap<>(); + private final LdioLdesClientConnectorApiCollection clientConnectorApis; + + public LdioLdesClientConnectorApiController(LdioLdesClientConnectorApiCollection clientConnectorApis) { + this.clientConnectorApis = clientConnectorApis; + } @PostMapping(path = "/{pipeline}/token") public ResponseEntity handleToken(@PathVariable("pipeline") String pipeline, @RequestBody String token) { - ofNullable(clientConnectorApis.get(pipeline)) + clientConnectorApis.get(pipeline) .orElseThrow(() -> new IllegalArgumentException("Not a valid pipeline")) .handleToken(token); return ResponseEntity.accepted().build(); @@ -28,14 +30,32 @@ public ResponseEntity handleToken(@PathVariable("pipeline") String pipel @PostMapping(path = "/{pipeline}/transfer") public ResponseEntity handleTransfer(@PathVariable("pipeline") String pipeline, @RequestBody String transfer) { return ResponseEntity.accepted() - .body(ofNullable(clientConnectorApis.get(pipeline)) + .body(clientConnectorApis.get(pipeline) .orElseThrow(() -> new IllegalArgumentException("Not a valid pipeline")) .handleTransfer(transfer)); } @EventListener void handleNewPipelines(LdesClientConnectorApiCreatedEvent connectorApiCreatedEvent) { - clientConnectorApis.put(connectorApiCreatedEvent.pipelineName(), connectorApiCreatedEvent.ldesClientConnectorApi()); + clientConnectorApis.add(connectorApiCreatedEvent.pipelineName(), connectorApiCreatedEvent.ldesClientConnectorApi()); + } + + @EventListener + void deletePipeline(PipelineDeletedEvent deletedEvent) { + clientConnectorApis.remove(deletedEvent.pipelineId()).shutdown(); + } + + @EventListener + public void handlePipelineStatusEvent(PipelineStatusEvent event) { + clientConnectorApis.get(event.pipelineId()).ifPresent(connectorApi -> { + switch (event.status()) { + case RUNNING -> connectorApi.resume(); + case HALTED -> connectorApi.pause(); + default -> { + // do nothing with the other status events + } + } + }); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java new file mode 100644 index 000000000..8e3124152 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java @@ -0,0 +1,13 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.collection; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; + +import java.util.Optional; + +public interface LdioLdesClientConnectorApiCollection { + Optional get(String pipeline); + + void add(String pipeline, LdioLdesClientConnectorApi ldioLdesClientConnectorApi); + + LdioLdesClientConnectorApi remove(String pipeline); +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java new file mode 100644 index 000000000..130ab4a4f --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollectionImpl.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.collection; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +@Component +public class LdioLdesClientConnectorApiCollectionImpl implements LdioLdesClientConnectorApiCollection { + private final Map clientConnectorApis = new HashMap<>(); + + @Override + public Optional get(String pipeline) { + return Optional.ofNullable(clientConnectorApis.get(pipeline)); + } + + @Override + public void add(String pipeline, LdioLdesClientConnectorApi ldioLdesClientConnectorApi) { + clientConnectorApis.put(pipeline, ldioLdesClientConnectorApi); + } + + @Override + public LdioLdesClientConnectorApi remove(String pipeline) { + return clientConnectorApis.remove(pipeline); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 247ea21e8..b6e57899c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTokenService; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTokenServiceLifecycle; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.MemoryTransferService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.valueobjects.EdcUrlProxy; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; @@ -40,7 +41,7 @@ public static class LdioClientConnectorConfigurator implements LdioInputConfigur public static final String PROXY_URL_REPLACEMENT = "proxy-url-replacement"; private final ApplicationEventPublisher eventPublisher; private final ObservationRegistry observationRegistry; - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(); + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private final RequestExecutor baseRequestExecutor = requestExecutorFactory.createNoAuthExecutor(); public LdioClientConnectorConfigurator(ApplicationEventPublisher eventPublisher, ObservationRegistry observationRegistry) { @@ -53,7 +54,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final String pipelineName = properties.getPipelineName(); final var connectorTransferUrl = properties.getProperty(CONNECTOR_TRANSFER_URL); final var transferService = new MemoryTransferService(baseRequestExecutor, connectorTransferUrl); - final var tokenService = new MemoryTokenService(transferService); + final var memoryTokenServiceLifecycle = new MemoryTokenServiceLifecycle(); + final var tokenService = new MemoryTokenService(transferService, memoryTokenServiceLifecycle); final var urlProxy = getEdcUrlProxy(properties); final var edcRequestExecutor = requestExecutorFactory.createEdcExecutor(baseRequestExecutor, tokenService, @@ -62,9 +64,9 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); var ldesClient = new LdioLdesClient(pipelineName, executor, observationRegistry, memberSupplier, applicationEventPublisher, keepState); - ldesClient.start(); eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); + ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java index 63742ba22..40b8bb8e2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java @@ -10,6 +10,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.ComponentScan; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; @@ -22,6 +23,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SpringBootTest(classes = LdioLdesClientConnectorApiController.class) +@ComponentScan(basePackages = "be.vlaanderen.informatievlaanderen.ldes.ldio.collection") @AutoConfigureMockMvc class LdioLdesClientConnectorTest { @@ -32,13 +34,12 @@ class LdioLdesClientConnectorTest { private final String endpoint = "endpoint"; private TransferService transferService; private TokenService tokenService; - private LdioLdesClient ldesClient; @BeforeEach void setup() { transferService = mock(TransferService.class); tokenService = mock(TokenService.class); - ldesClient = mock(LdioLdesClient.class); + final LdioLdesClient ldesClient = mock(LdioLdesClient.class); eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index dbf5e901d..3269b8375 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 66ac226f3..3e84d42e5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -35,12 +35,12 @@ public LdioLdesClient(String pipelineName, this.keepState = keepState; } - @SuppressWarnings("java:S2095") + @Override public void start() { + super.start(); final ExecutorService executorService = newSingleThreadExecutor(); executorService.submit(() -> { memberSupplier.init(); - this.starting(); this.run(); }); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index 2567d289d..52ac39679 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -1,6 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; @@ -37,7 +38,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecut ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { String pipelineName = properties.getPipelineName(); - RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(properties); + final var requestExecutorFactory = new RequestExecutorFactory(false); + final var requestExecutor = new LdioRequestExecutorSupplier(requestExecutorFactory).getRequestExecutor(properties); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, requestExecutor).getMemberSupplier(); final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, memberSupplier, diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index ca4045bff..efcd37f04 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index b2f28143c..9ef3d5073 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index af70a270b..df3ec9738 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index aa1074921..629338db6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java index 20528c6ff..896307ed1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioRepositoryMaterialiser.java @@ -9,7 +9,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.LocalDateTime; +import java.io.File; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ScheduledExecutorService; @@ -55,24 +56,30 @@ public void shutdown() { public synchronized void sendToMaterialiser() { if (!membersToCommit.isEmpty()) { - materialiser.processAsync(List.copyOf(membersToCommit)) + final List members = List.copyOf(membersToCommit); + materialiser.processAsync(members) .exceptionally(throwable -> { - handleException(throwable); + handleException(throwable, members); return null; }); membersToCommit.clear(); } } - private void handleException(Throwable throwable) { + private void handleException(Throwable throwable, List failedMembers) { log.atError().log(ERROR_TEMPLATE, "sendToMaterialiser", throwable.getMessage()); - final String fileName = "/tmp/materialisation/uncommitted-members-%s.ttl".formatted(LocalDateTime.now()); final Model uncommitedMembersModel = ModelFactory.createDefaultModel(); - membersToCommit.forEach(uncommitedMembersModel::add); - RDFWriter.source(uncommitedMembersModel) - .lang(Lang.TURTLE) - .output(fileName); - log.atError().log("Uncommitted members can be found in file: {}", fileName); + failedMembers.forEach(uncommitedMembersModel::add); + final File materialisationFolder = new File("materialisation"); + if (materialisationFolder.exists() || materialisationFolder.mkdir()) { + final String fileName = "uncommitted-members-%s.ttl".formatted(ZonedDateTime.now().toEpochSecond()); + final File uncommittedMembersFile = new File(materialisationFolder, fileName); + RDFWriter.source(uncommitedMembersModel).lang(Lang.TURTLE).output(uncommittedMembersFile.getPath()); + log.atError().log("Uncommitted members can be found in file: {}", uncommittedMembersFile.getAbsolutePath()); + } else { + final String uncommittedMembers = RDFWriter.source(uncommitedMembersModel).lang(Lang.TURTLE).asString(); + log.atError().log("Unable to commit the following members: \n {}", uncommittedMembers); + } } private void resetExecutor() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index c960299ad..8d4458479 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index 4bd80236e..2c108101b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index a6912d41e..ba2f75903 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index 0b3771733..eab79311b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 857cccba5..2c994abff 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index abeddf3f1..e4a658be0 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 6f30f2899..5c74d3382 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT 4.0.0 diff --git a/ldio.Dockerfile b/ldio.Dockerfile index 36621f7e5..c144efd2a 100644 --- a/ldio.Dockerfile +++ b/ldio.Dockerfile @@ -3,7 +3,7 @@ # # INSTALL MAVEN DEPENDENCIES # -FROM maven:3.8.5-amazoncorretto-17 AS builder +FROM maven:3.9.6-amazoncorretto-21 AS builder # MAVEN: application FROM builder AS app-stage @@ -13,7 +13,7 @@ RUN mvn clean install -DskipTests # # RUN THE APPLICATION # -FROM amazoncorretto:17-alpine-jdk +FROM amazoncorretto:21-alpine-jdk RUN adduser -D -u 2000 ldio USER ldio diff --git a/pom.xml b/pom.xml index 511ee3e21..1482f800d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT ldi-api @@ -37,9 +37,10 @@ - 17 - 17 - 17 + 21 + 21 + 21 + 21 3.5.0 3.2.1 @@ -54,7 +55,6 @@ false false - 1.9.0 4.10.0 1.4 @@ -71,14 +71,13 @@ 3.24.2 5.10.0 4.2.0 - 1.14.11 + 1.14.12 2.35.1 5.5.0 5.5.0 1.19.6 1.19.0 2.0.7 - 1.14.7 1.4.11 0.8.10 @@ -120,25 +119,6 @@ - - net.revelc.code - impsort-maven-plugin - ${impsort-maven-plugin.version} - - true - *, java.,javax.,com. - * - true - - - - sort-imports - - sort - - - - org.apache.maven.plugins maven-surefire-plugin @@ -165,6 +145,7 @@ integration-test + verify