Merge pull request #424 from Informatievlaanderen/feat/default-bucket

feat: default bucket test

cypress/e2e/server.basics.feature

@@ -57,4 +57,16 @@ Feature: LDES Server Basic Functionality
     And context 'tests/035.relative-urls' is started
     And the LDES server is available and configured
     When I start the LDES Client 'LDIO' workbench
-    Then the sink contains 5 members in collection 'mobility-hindrances'
+    Then the sink contains 5 members in collection 'mobility-hindrances'
+
+  @test-039 @default-fragment
+  Scenario: 039: Verify server puts non-fragmentable members in a default fragment
+    Given the members are stored in database 'bustang'
+    And context 'tests/039.default-bucket' is started
+    And the LDES server is available and configured
+    When I start the LDES Client 'LDIO' workbench
+    Then the sink contains 5 members in collection 'mobility-hindrances'
+    Then the 'time' view has a relation to the default fragment
+    And the 'ref' view has a relation to the default fragment
+    And the 'geo' view has a relation to the default fragment
+    And the 'mixed' view has a relation to the default fragment

cypress/support/ldes/fragment.ts

@@ -38,6 +38,7 @@ export class Fragment extends UrlResponse {
     }
 
     expectSingleRelationOf(type: string): Relation {
+        cy.log(type)
         const relations = this.relations.filter(x => x.type === tree.prefix(type));
         expect(relations.length).to.equal(1);
         return relations[0];

cypress/support/step_definitions/common_step_definitions.ts

@@ -67,6 +67,12 @@ export function range(start: number, end: number) {
     return new Array(end - start + 1).fill(start).map((_, i) => i + 1);
 }
 
+export function obtainViewWithDefaultFragment(ldes: string, view: string) {
+    return server.getLdes(ldes)
+        .then(ldes => new Fragment(ldes.viewUrl(view).replace("ldes-server", "localhost")))
+        .then(view => waitForFragment(view, x => x.relations.filter(relation => relation.type == "https://w3id.org/tree#Relation").length >= 1, 'have 1 default relation'));
+}
+
 export function obtainRootFragment(ldes: string, view: string) {
     return server.getLdes(ldes)
         .then(ldes => new Fragment(ldes.viewUrl(view)))

cypress/support/step_definitions/server.basics.feature.ts

@@ -2,7 +2,7 @@
 
 import { When, Then } from "@badeball/cypress-cucumber-preprocessor";
 import { Fragment } from '../ldes';
-import { server, waitForFragment, obtainRootFragment, byPage } from "./common_step_definitions";
+import { server, waitForFragment, obtainRootFragment, byPage, obtainViewWithDefaultFragment } from "./common_step_definitions";
 
 let ldesResponse: Cypress.Response<any>;
 let viewResponse: Cypress.Response<any>;
@@ -99,3 +99,7 @@ Then('the first page contains {int} members', (count: number) => {
     waitForFragment(rootFragment, x => x.memberCount === count, `have member count equal ${count}`);
 })
 
+Then('the {string} view has a relation to the default fragment', (view: string) => {
+    obtainViewWithDefaultFragment("mobility-hindrances", view);
+})
+

tests/039.default-bucket/.env

@@ -0,0 +1,2 @@
+# NOTE: empty, add any environment overrides here, e.g.
+# USECASE_NAME=my-use-case

tests/039.default-bucket/README.md

@@ -0,0 +1,71 @@
+# LDES Server makes unfragmentable members available in a default fragment
+The test verifies that the LDES server can makes all received members available, even if they can not be assigned to a fragment.
+
+The server (http://localhost:8080) is seeded by a subset of the mobility-hindrances dataset.
+
+## Test Setup
+> **Note**: if needed, copy the [environment file (.env)](./.env) to a personal file (e.g. `user.env`) and change the settings as needed. If you do, you need to add ` --env-file user.env` to each `docker compose` command.
+
+Run all systems except the workflow by executing the following (bash) command:
+```bash
+docker compose up -d
+```
+
+## Test Execution
+1. Seed the LDES Server with a collection and a view and i send 5 members:
+    ```bash
+   chmod +x ./config/seed.sh
+   sh ./config/seed.sh
+   ```
+    To verify that the [server](http://localhost:8080) is correctly seeded you can run this command: 
+    ```bash
+    curl http://localhost:8080/mobility-hindrances
+    ```
+
+2. Verify the root fragment of the timebased view has a relation to the default fragment and to the year=2023 fragment:
+    ```bash
+    curl http://localhost:8080/mobility-hindrances/time
+    ```
+
+3. Verify the default fragment contains exactly 2 members:
+    ```bash
+    curl 'http://localhost:8080/mobility-hindrances/time?year=unknown&pageNumber=1'
+    ```
+
+4. Verify the root fragment of the geospatial view has a relation to the default fragment and to the year=2023 fragment:
+    ```bash
+    curl http://localhost:8080/mobility-hindrances/geo
+    ```
+
+5. Verify the default fragment contains exactly 2 members:
+    ```bash
+    curl 'http://localhost:8080/mobility-hindrances/geo?tile=unknown&pageNumber=1'
+    ```
+
+6. Verify the root fragment of the reference view has a relation to the default fragment and to the year=2023 fragment:
+    ```bash
+    curl http://localhost:8080/mobility-hindrances/ref
+    ```
+
+7. Verify the default fragment contains exactly 2 members:
+    ```bash
+    curl 'http://localhost:8080/mobility-hindrances/ref?version=unknown&pageNumber=1'
+    ```
+
+8. Verify the root fragment of the combined view has a relation to the default fragment and to the year=2023 fragment:
+    ```bash
+    curl http://localhost:8080/mobility-hindrances/mixed
+    ```
+
+9. Verify the default fragment contains exactly 2 members:
+    ```bash
+    curl http://localhost:8080/mobility-hindrances/mixed?year=unknown&tile=unknown&version=unknown&pageNumber=1
+    ```
+
+
+## Test Teardown
+To stop all systems use:
+```bash
+docker compose rm -s -f -v ldio-workbench
+docker compose --profile delay-started down
+```

tests/039.default-bucket/config/ldes.ttl

@@ -0,0 +1,15 @@
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix dcterms: <http://purl.org/dc/terms/> .
+@prefix tree: <https://w3id.org/tree#>.
+@prefix sh:   <http://www.w3.org/ns/shacl#> .
+@prefix prov:   <http://www.w3.org/ns/prov#> .
+@prefix ex:   <http://example.org/> .
+
+
+<mobility-hindrances> a ldes:EventStream ;
+    ldes:timestampPath prov:generatedAtTime ;
+    ldes:versionOfPath dcterms:isVersionOf ;
+    tree:shape [
+        sh:closed false;
+        a sh:NodeShape ;
+    ] .

tests/039.default-bucket/config/members/member0.ttl

@@ -0,0 +1,36 @@
+@prefix dc11: <http://purl.org/dc/elements/1.1/> .
+@prefix dc: <http://purl.org/dc/terms/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix ns0: <http://www.w3.org/ns/adms#> .
+@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
+@prefix ns1: <https://gipod.vlaanderen.be/ns/gipod#> .
+@prefix ns2: <http://www.w3.org/ns/> .
+@prefix prov: <http://www.w3.org/ns/prov#> .
+@prefix ns3: <https://data.vlaanderen.be/ns/mobiliteit#Inname.> .
+@prefix ns4: <https://data.vlaanderen.be/ns/mobiliteit#> .
+@prefix ns5: <http://data.europa.eu/m8g/> .
+@prefix ns6: <http://www.opengis.net/ont/geosparql#> .
+@prefix org: <http://www.w3.org/ns/org#> .
+
+<https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/1/0>
+  dc11:contributor <https://private-api.gipod.beta-vlaanderen.be/api/v1/organisations/a7eadbde-86a6-076c-bd2f-7f5596ba3730> ;
+  dc:description "omschrijving" ;
+  dc:isVersionOf <https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/1> ;
+  dc:modified "2022-05-20T09:58:15.8646433Z"^^xsd:dateTime ;
+  a <https://data.vlaanderen.be/ns/mobiliteit#Mobiliteitshinder> ;
+  ns0:identifier [
+    a ns0:Identifier ;
+    skos:notation "1"^^ns1:gipodId ;
+    ns0:schemaAgency "https://gipod.vlaanderen.be"@nl-be
+  ] ;
+  ns2:adms1Notes "MobilityHindranceZoneWasAdded"@nl-be ;
+  prov:generatedAtTime "2023-11-09T10:24:23.8646433Z"^^xsd:dateTime ;
+  ns3:status <https://private-api.gipod.beta-vlaanderen.be/api/v1/taxonomies/statuses/0a4ee99b-8b8a-47c8-913f-117220febee0> ;
+  ns4:beheerder <https://private-api.gipod.beta-vlaanderen.be/api/v1/organisations/a7eadbde-86a6-076c-bd2f-7f5596ba3730> ;
+  ns4:periode [
+    ns5:endTime "2022-05-27T17:00:00Z"^^xsd:dateTime ;
+    ns5:startTime "2022-05-27T07:00:00Z"^^xsd:dateTime ;
+    a ns5:PeriodOfTime
+  ] ;
+  ns4:zone <https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/1/zones/2af888b0-de0c-48c2-956b-8b36d01c1232> ;
+  ns1:gipodId 1 .

tests/039.default-bucket/config/seed.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+export SCRIPT_PATH=$(dirname -- "$( readlink -f -- "${BASH_SOURCE:-$0}"; )")
+
+curl --fail -X POST 'http://localhost:8080/admin/api/v1/eventstreams' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/ldes.ttl"
+code=$?
+if [ $code != 0 ]
+    then exit $code
+fi
+
+curl --fail -X POST 'http://localhost:8080/admin/api/v1/eventstreams/mobility-hindrances/views' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/view-time.ttl"
+code=$?
+if [ $code != 0 ]
+    then exit $code
+fi
+curl --fail -X POST 'http://localhost:8080/admin/api/v1/eventstreams/mobility-hindrances/views' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/view-geo.ttl"
+code=$?
+if [ $code != 0 ]
+    then exit $code
+fi
+curl --fail -X POST 'http://localhost:8080/admin/api/v1/eventstreams/mobility-hindrances/views' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/view-ref.ttl"
+code=$?
+if [ $code != 0 ]
+    then exit $code
+fi
+curl --fail -X POST 'http://localhost:8080/admin/api/v1/eventstreams/mobility-hindrances/views' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/view-mixed.ttl"
+code=$?
+if [ $code != 0 ]
+    then exit $code
+fi
+
+i=0
+while [ $i -lt 5 ];
+do
+    curl --fail -X POST 'http://localhost:8080/mobility-hindrances' -H 'Content-Type: text/turtle' -d "@$SCRIPT_PATH/members/member$i.ttl"
+    code=$?
+    i=$((i+1))
+    if [ $code != 0 ]
+        then exit $code
+    fi
+done
+
+

tests/039.default-bucket/config/view-geo.ttl

@@ -0,0 +1,15 @@
+@prefix tree: <https://w3id.org/tree#> .
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix example: <http://example.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+<geo> a ldes:View ;
+  tree:viewDescription [
+    tree:fragmentationStrategy ([
+                                        a tree:GeospatialFragmentation ;
+                                        tree:maxZoom 15 ;
+                                        tree:fragmentationPath <http://www.opengis.net/ont/geosparql#asWKT> ;
+                                    ]) ;
+    tree:pageSize "10"^^<http://www.w3.org/2001/XMLSchema#int> ;
+  ] .

tests/039.default-bucket/config/view-mixed.ttl

@@ -0,0 +1,26 @@
+@prefix tree: <https://w3id.org/tree#> .
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix example: <http://example.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+
+<mixed> a ldes:View ;
+  tree:viewDescription [
+    tree:fragmentationStrategy ([
+                                        a tree:GeospatialFragmentation ;
+                                        tree:maxZoom 15 ;
+                                        tree:fragmentationPath <http://www.opengis.net/ont/geosparql#asWKT> ;
+                                    ]
+                                    [
+                                        a tree:ReferenceFragmentation ;
+                                        tree:fragmentationPath "<http://purl.org/dc/elements/1.1/creator>" ;
+                                        tree:fragmentationKey "version" ;
+                                    ]
+                                    [
+                                        a tree:HierarchicalTimeBasedFragmentation ;
+                                        tree:maxGranularity "month" ;
+                                        tree:fragmentationPath dc:created ;
+                                    ]) ;
+    tree:pageSize "10"^^<http://www.w3.org/2001/XMLSchema#int> ;
+  ] .

tests/039.default-bucket/config/view-ref.ttl

@@ -0,0 +1,15 @@
+@prefix tree: <https://w3id.org/tree#> .
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix example: <http://example.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+<ref> a ldes:View ;
+  tree:viewDescription [
+    tree:fragmentationStrategy ([
+                                        a tree:ReferenceFragmentation ;
+                                        tree:fragmentationPath "<http://purl.org/dc/elements/1.1/creator>" ;
+                                        tree:fragmentationKey "version" ;
+                                    ]) ;
+    tree:pageSize "10"^^<http://www.w3.org/2001/XMLSchema#int> ;
+  ] .

tests/039.default-bucket/config/view-time.ttl

@@ -0,0 +1,16 @@
+@prefix tree: <https://w3id.org/tree#> .
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix example: <http://example.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+
+<time> a ldes:View ;
+  tree:viewDescription [
+    tree:fragmentationStrategy ([
+                                        a tree:HierarchicalTimeBasedFragmentation ;
+                                        tree:maxGranularity "month" ;
+                                        tree:fragmentationPath dc:created ;
+                                    ]) ;
+    tree:pageSize "10"^^<http://www.w3.org/2001/XMLSchema#int> ;
+  ] .

tests/039.default-bucket/docker-compose.yml

@@ -0,0 +1,53 @@
+version: '3.3'
+services:
+
+  ldes-server:
+    image: ghcr.io/informatievlaanderen/ldes-server:${LDES_SERVER_TAG:-latest}
+    container_name: ldes-server
+    environment:
+      - SIS_DATA=/tmp
+    volumes:
+      - ./ldes-server.config.yml:/application.yml:ro
+    ports:
+      - ${LDES_SERVER_PORT:-8080}:8080
+    depends_on:
+      - ldes-mongodb
+    networks:
+      - ldes
+
+  test-message-sink:
+    image: ghcr.io/informatievlaanderen/test-message-sink:${TEST_MESSAGE_SINK_TAG:-latest}
+    container_name: test-message-sink
+    environment:
+      - SILENT=false
+      - MEMORY=false
+      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+      - DATABASE_NAME=testSink
+      - COLLECTION_NAME=mobility-hindrances
+      - MEMBER_TYPE=https://data.vlaanderen.be/ns/mobiliteit#Mobiliteitshinder
+    ports:
+      - ${TEST_MESSAGE_SINK_PORT:-9003}:80
+    networks:
+      - ldes
+
+  ldio-workbench:
+    container_name: ldio-workbench
+    image: ${LDI_ORCHESTRATOR:-ghcr.io/informatievlaanderen/ldi-orchestrator}:${LDI_ORCHESTRATOR_TAG:-latest}
+    volumes:
+      - ./ldio-workflow.config.yml:/ldio/application.yml:ro
+    networks:
+      - ldes
+    profiles:
+      - delay-started
+
+  ldes-mongodb:
+    container_name: ldes-mongodb
+    image: mongo:${MONGODB_TAG:-latest}
+    ports:
+      - ${MONGODB_PORT:-27017}:27017
+    networks:
+      - ldes
+
+networks:
+  ldes:
+    name: ${USECASE_NAME:-default-bucket}_network

tests/039.default-bucket/ldes-server.config.yml

@@ -0,0 +1,17 @@
+mongock:
+  migration-scan-package: VSDS
+springdoc:
+  swagger-ui:
+    path: /v1/swagger
+management:
+  tracing:
+    enabled: false
+ldes-server:
+  host-name: http://ldes-server:8080
+  use-relative-url: false
+spring:
+  data:
+    mongodb:
+      database: bustang
+      host: ldes-mongodb
+      port: 27017

tests/039.default-bucket/ldio-workflow.config.yml

@@ -0,0 +1,14 @@
+orchestrator:
+  pipelines:
+    - name: simulator-to-sink-pipeline
+      description: "This pipeline uses an LDES client to read an existing LDES and send the members as N-Quads to a sink application"
+      input:
+        name: Ldio:LdioLdesClient
+        config:
+          url: http://ldes-server:8080/mobility-hindrances/time
+      outputs:
+        - name: Ldio:LdioHttpOut
+          config:
+            endpoint: http://test-message-sink/member
+            rdf-writer:
+              content-type: application/n-quads