From f60b775072ab9af1787807a098f5d01ea1a832e0 Mon Sep 17 00:00:00 2001 From: smessie Date: Wed, 6 Mar 2024 11:35:14 +0100 Subject: [PATCH 1/3] fix: Correctly wait upon depending services being started --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 17e476e..6fcce7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,8 +26,10 @@ services: ports: - 8084:8080 depends_on: - - postgresql - - rdf4j-server + postgresql: + condition: service_healthy + rdf4j-server: + condition: service_healthy networks: - demonstrator-network From 9120f7bba8fe0ea390b27f85041e276fb64d16a7 Mon Sep 17 00:00:00 2001 From: smessie Date: Wed, 6 Mar 2024 11:36:10 +0100 Subject: [PATCH 2/3] fix: Map to correct port of process in container --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6fcce7f..32f8dd3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: ghcr.io/informatievlaanderen/ldi-orchestrator:latest container_name: demonstrator-data-provider ports: - - 8082:8080 + - 8082:8082 volumes: - ./docker-compose/data-provider.config.yml:/ldio/application.yml:ro - ./docker-compose/rml:/ldio/rml:ro From da6a08b2e47172b21aa50b8d040c9fe28e861db2 Mon Sep 17 00:00:00 2001 From: smessie Date: Wed, 6 Mar 2024 11:37:30 +0100 Subject: [PATCH 3/3] fix: Update outdated configs --- docker-compose/data-provider.config.yml | 58 +++++++++++++------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/docker-compose/data-provider.config.yml b/docker-compose/data-provider.config.yml index f4cafef..7f1d6c4 100644 --- a/docker-compose/data-provider.config.yml +++ b/docker-compose/data-provider.config.yml @@ -4,36 +4,37 @@ orchestrator: pipelines: - name: gipod input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + name: Ldio:LdesClientConnector config: url: https://private-api.gipod.vlaanderen.be/api/v1/ldes/mobility-hindrances?generatedAtTime=2024-10-05T10:28:37.923Z - sourceFormat: application/ld+json + source-format: application/ld+json outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + - name: Ldio:RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: test named-graph: http://gipod - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + - name: Ldio:HttpOut config: - content-type: application/n-quads + rdf-writer: + content-type: application/n-quads endpoint: http://host.docker.internal:8084/api/gipod/members - name: verkeersmetingen-pipeline description: "Simple http in, version creation, http out pipeline allowing to pause output." input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + name: Ldio:HttpInPoller config: url: http://miv.opendata.belfla.be/miv/verkeersdata interval: PT10S adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + name: Ldio:RmlAdapter config: mapping: /ldio/rml/verkeersmetingen.ttl transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + - name: Ldio:SparqlConstructTransformer config: query: /ldio/sparql/meting.to-oslo.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator + - name: Ldio:VersionObjectCreator config: member-type: https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting delimiter: "/" @@ -41,75 +42,78 @@ orchestrator: generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime versionOf-property: http://purl.org/dc/terms/isVersionOf outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + - name: Ldio:RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: test named-graph: http://verkeersmeting - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + - name: Ldio:HttpOut config: endpoint: http://host.docker.internal:8084/api/verkeersmeting/members - content-type: application/n-quads + rdf-writer: + content-type: application/n-quads - name: stations-pipeline description: "HTTP polling, OSLO transformation, version creation & HTTP sending." input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + name: Ldio:HttpInPoller config: url: https://api.blue-bike.be/pub/location interval: PT2M adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + name: Ldio:RmlAdapter config: mapping: /ldio/rml/stations.ttl outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + - name: Ldio:RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: test named-graph: http://bluebikes - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + - name: Ldio:HttpOut config: endpoint: http://host.docker.internal:8084/api/bluebikes/members - content-type: application/n-quads + rdf-writer: + content-type: application/n-quads - name: crowdscan-observations-pipeline input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + name: Ldio:LdesClientConnector config: url: https://azure.crowdscan.be/ldes-scewc/observations transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + - name: Ldio:SparqlConstructTransformer config: query: ./sparql/crowdscan.add-query.rq infer: true - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher + - name: Ldio:HttpEnricher config: url-property-path: https://crowdscan.be/ns/HttpRequest.url body-property-path: https://crowdscan.be/ns/HttpRequest.body http-method-property-path: https://crowdscan.be/ns/HttpRequest.method header-property-path: https://crowdscan.be/ns/HttpRequest.header adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + name: Ldio:RdfAdapter + - name: Ldio:SparqlConstructTransformer config: query: ./sparql/crowdscan.remove-query.rq infer: false outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + - name: Ldio:RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: test named-graph: http://crowdscan - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + - name: Ldio:HttpOut config: endpoint: http://host.docker.internal:8084/api/crowdscan/members - content-type: application/n-quads + rdf-writer: + content-type: application/n-quads - name: crowdscan-locations-pipeline input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + name: Ldio:LdesClientConnector config: url: https://azure.crowdscan.be/ldes-scewc/zones outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + - name: Ldio:RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: locations