Merge pull request #419 from Informatievlaanderen/feat/ldes-discovere…

…r-e2e

Feat/ldes discoverer e2e

CODEOWNERS

@@ -1 +1 @@
-*       @Yalz @rorlic @WLefever-Cegeka @pj-cegeka @Tomvbe
+*       @Yalz @rorlic @pj-cegeka @Tomvbe @jobulcke

cypress.config.ts

@@ -60,6 +60,9 @@ async function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.PluginC
   const parkAndRideDataFolder = './data/parkAndRide';
   config.env.parkAndRide = getFiles(parkAndRideDataFolder).map(x => `${parkAndRideDataFolder}/${x}`);
 
+  const geomobilityDataFolder = './data/geomobility';
+  config.env.geomobility = getFiles(geomobilityDataFolder).map(x => `${geomobilityDataFolder}/${x}`);
+
   // read user environment file
   function parseEnvironmentFile(filePath: string) {
     const content = filePath && fs.existsSync(filePath) && fs.readFileSync(filePath, 'utf-8');

cypress/e2e/discoverer.feature

@@ -0,0 +1,10 @@
+@discoverer
+Feature: LDI LDES Discoverer
+
+  @test-038 @discover @geomobility
+  Scenario: 038: LDES Discoverer Can Discover The Structure of an LDES
+    Given context 'tests/038.discover-ldes-structure' is started
+    And I have aliased the 'geomobility' simulator data set
+    When I start the LDES Discoverer
+    Then the LDES structure contains 10 relations
+    And the LDES structure is equal to "expected-output.txt"

cypress/support/services/ldi-ldes-discoverer.ts

@@ -0,0 +1,76 @@
+/// <reference types="cypress" />
+
+import {CanCheckAvailability} from "./interfaces";
+import {timeouts} from "../common";
+
+export class LdiLdesDiscoverer implements CanCheckAvailability {
+    constructor(private _serviceName?: string) {
+    }
+
+    public get serviceName() {
+        return this._serviceName || 'ldes-discoverer';
+    }
+
+    protected get availabilityMessage() {
+        return "Started Application in";
+    }
+
+    private containerLogIncludes(containerId: string, includeString: string) {
+        return cy.exec(`docker logs ${containerId}`).then(result => result.stdout.includes(includeString));
+    }
+
+    waitForDockerLog(includeString: string) {
+        return cy.exec(`docker ps -f "name=${this.serviceName}$" -q`)
+            .then(result => {
+                const containerId = result.stdout;
+                return cy.waitUntil(() => this.containerLogIncludes(containerId, includeString), {
+                    timeout: timeouts.slowAction,
+                    interval: timeouts.slowCheck,
+                    errorMsg: `Timed out waiting for container '${this.serviceName}' log to include '${includeString}'`
+                });
+            });
+    }
+
+    waitAvailable() {
+        this.waitForDockerLog(this.availabilityMessage);
+    }
+
+    getContainerId() {
+        return cy.exec(`docker ps -qaf "name=${this.serviceName}$" -q`)
+            .then(result => result.stdout)
+    }
+
+    extractLogs(containerId: string) {
+        return cy.exec(`docker logs ${containerId}`).then(result => result.stdout)
+    }
+
+    hasCount(containerId: string, count: number): any {
+        return this.extractLogs(containerId).then(logs => {
+            const totalOfStr = "contains a total of ";
+            const startIndex = logs.indexOf(totalOfStr) + totalOfStr.length;
+            const endIndex = logs.indexOf(" relations:");
+            const actualCount = Number.parseInt(logs.substring(startIndex, endIndex).trim());
+            cy.log(`Actual count: ${actualCount}`).then(() => actualCount === count)
+        });
+    }
+
+    checkRelationCount(count: number) {
+        return this.getContainerId()
+            .then(containerId => {
+                return cy.waitUntil(() => this.hasCount(containerId, count), {
+                    timeout: timeouts.ready,
+                    interval: timeouts.check,
+                    errorMsg: `Timed out waiting for relation count to be ${count}`
+                })
+            })
+    }
+
+    checkOutputStructure(expectedOutputFilePath: string) {
+        return cy.exec(`cat ${expectedOutputFilePath}`)
+            .then(result => {
+                this.getContainerId().then(containerId => {
+                    this.containerLogIncludes(containerId, result.stdout)
+                })
+            })
+    }
+}

cypress/support/step_definitions/common_step_definitions.ts

@@ -1,12 +1,18 @@
-import { After, Given, When, Then, Before } from "@badeball/cypress-cucumber-preprocessor";
-import { DockerCompose, DockerComposeOptions, EnvironmentSettings, checkSuccess, timeouts } from "..";
+import {After, Before, Given, Then, When} from "@badeball/cypress-cucumber-preprocessor";
+import {checkSuccess, DockerCompose, DockerComposeOptions, EnvironmentSettings, timeouts} from "..";
 import {
-    LdesWorkbenchNiFi, LdesServerSimulator, TestMessageSink,
-    MongoRestApi, TestMessageGenerator, LdesServer, LdesWorkbenchLdio
+    LdesServer,
+    LdesServerSimulator,
+    LdesWorkbenchLdio,
+    LdesWorkbenchNiFi,
+    MongoRestApi,
+    TestMessageGenerator,
+    TestMessageSink,
 } from "../services";
-import { Gtfs2Ldes } from "../services/gtfs2ldes";
-import { Fragment } from "../ldes";
-import { LdesClientWorkbench } from "../services/ldes-client-workbench";
+import {Gtfs2Ldes} from "../services/gtfs2ldes";
+import {Fragment} from "../ldes";
+import {LdesClientWorkbench} from "../services/ldes-client-workbench";
+import {LdiLdesDiscoverer} from "../services/ldi-ldes-discoverer";
 
 let testContext: any;
 const ldesMemberCollection = 'ingest_ldesmember';
@@ -22,6 +28,7 @@ export const mongo = new MongoRestApi('http://localhost:9019');
 export const jsonDataGenerator = new TestMessageGenerator();
 export const server = new LdesServer('http://localhost:8080');
 export const gtfs2ldes = new Gtfs2Ldes();
+export const ldesDiscoverer = new LdiLdesDiscoverer();
 
 export const byPage = 'by-page';
 
@@ -59,7 +66,7 @@ export function ensureRelationCount(fragment: Fragment, amount: number) {
 export function setTargetUrl(targeturl: string) {
     const command = `echo ${targeturl} > ${testContext.testPartialPath}/data/TARGETURL`;
     return cy.log(command)
-        .exec(command, { log: true, failOnNonZeroExit: false })
+        .exec(command, {log: true, failOnNonZeroExit: false})
         .then(result => checkSuccess(result).then(success => expect(success).to.be.true))
 }
 
@@ -81,8 +88,12 @@ export function obtainRootFragment(ldes: string, view: string) {
 
 export function waitForFragment(fragment: Fragment, condition: (x: Fragment) => boolean, message: string) {
     return cy.waitUntil(() =>
-        fragment.visit().then(fragment => condition(fragment)),
-        { timeout: timeouts.fastAction, interval: timeouts.check, errorMsg: `Timed out waiting for ${fragment.url} to ${message}.` }).then(() => fragment);
+            fragment.visit().then(fragment => condition(fragment)),
+        {
+            timeout: timeouts.fastAction,
+            interval: timeouts.check,
+            errorMsg: `Timed out waiting for ${fragment.url} to ${message}.`
+        }).then(() => fragment);
 }
 
 export function clientConnectorFailsOnStatusCode(code: number) {
@@ -171,7 +182,8 @@ Given('the {string} workbench is available', (workbench) => {
             workbenchLdio.waitAvailable();
             break;
         }
-        default: throw new Error(`Unknown workbench '${workbench}'`);
+        default:
+            throw new Error(`Unknown workbench '${workbench}'`);
     }
 })
 
@@ -193,7 +205,8 @@ When('I start the LDES Client {string} workbench', (workbench) => {
             createAndStartService(clientWorkbench.serviceName).then(() => clientWorkbench.waitAvailable());
             break;
         }
-        default: throw new Error(`Unknown workbench '${workbench}'`);
+        default:
+            throw new Error(`Unknown workbench '${workbench}'`);
     }
 })
 
@@ -207,7 +220,8 @@ When('I start the {string} workbench', (workbench) => {
             createAndStartService(workbenchLdio.serviceName).then(() => workbenchLdio.waitAvailable());
             break;
         }
-        default: throw new Error(`Unknown workbench '${workbench}'`);
+        default:
+            throw new Error(`Unknown workbench '${workbench}'`);
     }
 })
 
@@ -223,7 +237,8 @@ When('I pause the {string} pipeline on the {string} workbench', (pipeline: strin
             workbenchLdio.pause(pipeline);
             break;
         }
-        default: throw new Error(`Unknown workbench '${workbench}'`);
+        default:
+            throw new Error(`Unknown workbench '${workbench}'`);
     }
 })
 
@@ -239,10 +254,15 @@ When('I resume the {string} pipeline on the {string} workbench', (pipeline: stri
             workbenchLdio.resume(pipeline);
             break;
         }
-        default: throw new Error(`Unknown workbench '${workbench}'`);
+        default:
+            throw new Error(`Unknown workbench '${workbench}'`);
     }
 })
 
+When('I start the LDES Discoverer', () => {
+    createAndStartService(ldesDiscoverer.serviceName);
+})
+
 export function createAndStartService(service: string, additionalEnvironmentSettings?: EnvironmentSettings) {
     return dockerCompose.create(service, additionalEnvironmentSettings)
         .then(() => dockerCompose.start(service, additionalEnvironmentSettings));
@@ -253,7 +273,7 @@ export function stopAndRemoveService(service: string) {
 }
 
 When('I start the JSON Data Generator', () => {
-    createAndStartService(jsonDataGenerator.serviceName, { JSON_DATA_GENERATOR_SILENT: false })
+    createAndStartService(jsonDataGenerator.serviceName, {JSON_DATA_GENERATOR_SILENT: false})
         .then(() => jsonDataGenerator.waitAvailable());
 })
 
@@ -281,8 +301,12 @@ export function waitUntilMemberCountStable() {
     let previousCount: number;
     currentMemberCount().then(count => previousCount = count).then(count => cy.log(`Previous count: ${count}`));
     return cy.waitUntil(() =>
-        currentMemberCount().then(count => cy.log(`Current count: ${count}`).then(() => count === previousCount ? true : (previousCount = count, false))),
-        { timeout: timeouts.fastAction, interval: timeouts.check, errorMsg: `Timed out waiting for database '${testContext.database}' ingest count to retain the same (last: ${previousCount})` }
+            currentMemberCount().then(count => cy.log(`Current count: ${count}`).then(() => count === previousCount ? true : (previousCount = count, false))),
+        {
+            timeout: timeouts.fastAction,
+            interval: timeouts.check,
+            errorMsg: `Timed out waiting for database '${testContext.database}' ingest count to retain the same (last: ${previousCount})`
+        }
     );
 }
 
@@ -331,3 +355,10 @@ Then('the LDES member count increases', () => {
             (actual, expected) => actual > expected));
 })
 
+Then("the LDES structure contains {int} relations", (relationCount: number) =>
+    ldesDiscoverer.checkRelationCount(relationCount)
+)
+
+Then("the LDES structure is equal to {string}", (expectedOutputFileName: string) => {
+    ldesDiscoverer.checkOutputStructure(`${testContext.testPartialPath}/data/${expectedOutputFileName}`)
+})