Merge pull request #18 from meaningfy-ws/feature/mdr-69

New Diff templates and other things
meaningfy-ws · Oct 7, 2020 · 6def757 · 6def757
2 parents 76d4a43 + c791e14
commit 6def757
Show file tree

Hide file tree

Showing 69 changed files with 115,940 additions and 203 deletions.
diff --git a/.gitignore b/.gitignore
@@ -131,7 +131,12 @@ dmypy.json
 data-dev/
 build/
 
-# projest related
+# project related
 cucumber_report.json
 junit_report.xml
-*.ldb
+*.ldb
+/templates/diff/o/
+/templates/itb-shacl-report/o/
+/tests/test_data/validation-reports/simple/results/
+/tests/test_data/validation-reports/simple/tests/
+/tests/test_data/validation-reports/treaties-ap/results/
diff --git a/setup.py b/setup.py
@@ -71,4 +71,7 @@ def find_version(filename):
         "Natural Language :: English",
     ],
     python_requires='>=3.6',
+    entry_points={
+        "console_scripts": ["mkreport=eds4jinja2.entrypoints.cli.main:build_report"],
+    },
 )
diff --git a/templates/TODO.md → templates/diff/TODO.md b/templates/TODO.md → templates/diff/TODO.md
@@ -1,3 +1,11 @@
+# Current implementation status
+
+* Current template provides account in terms of *additions* and *deletions*, in teh future this can evolve to account for/detect *replacements, modifications and  movements*.
+* Further adjustments are needed to synchronise with VB3-SKOS-AP or SKOS-AP (to be discussed what templates are needed).   
+* Execution time ~ 5 minutes on Corporate Bodies table (queries for version and status seem to be slow)
+
+# Initial requirements
+
 We can aggregate based on concepts, on change type, on changed properties.
 
 The set of questions to answer in the diff template are:
@@ -10,7 +18,7 @@ The set of questions to answer in the diff template are:
 
 For the aforementioned 3,4 and 5 we must take into account that "simple/direct" and the reified version of these properties. We will start with the simple version at first and build on that. 
 
-Further reading:
+# Further reading:
 
 https://www.w3.org/TR/skos-reference/
 

diff --git a/templates/diff/config.json b/templates/diff/config.json
@@ -2,8 +2,8 @@
     "template": "main.html",
     "conf":
     {
-        "default_endpoint" : "http://localhost:3030/eurovoc-fragment/sparql",
-        "title": "Dataset Fingerprint Report (Diff)",
+        "default_endpoint" : "http://localhost:3030/cob/sparql",
+        "title": "RDF Diff report",
         "type": "report",
         "author": "Meaningfy",
         "ns_file": "data/prefix.csv"

diff --git a/templates/diff/templates/layout.html b/templates/diff/templates/layout.html
@@ -3,7 +3,7 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
-    <title>Proof of Concept Report</title>
+    <title>{{ conf.title }}</title>
 
     <meta name="description" content="">
     <meta name=viewport content="width=device-width, initial-scale=1">
@@ -41,7 +41,7 @@
     <div class="row"></div>
     <div class="row">
         <div class="starter column">
-            <h1 class="ui center aligned header" id="skip-toc">RDF Diff proof of concept</h1>
+            <h1 class="ui center aligned header" id="skip-toc">{{ conf.title }}</h1>
             {% block content %}
                 < empty >
             {% endblock %}

diff --git a/templates/diff/templates/main.html b/templates/diff/templates/main.html
@@ -6,65 +6,85 @@
     {# Read the and invert the namespace/prefix definitions for substitution #}
     {% set prefixes, error = from_file(conf.template_path+'/prefixes.json').fetch_tree() %}
     {% set inverted_prefixes = invert_dict(prefixes) %}
-
+
+    <p>This report is automatically generated from the RDF data and aims at presenting the difference between
+        two versions of an authority table maintained in VocBench3 following VB3-SRC-AP application profile.</p>
+    <p>The report is primarily focused on the concept instances (skos:Concept), and does not cover other classes as they
+        may occur (e.g. skos:ConceptScheme, skosxl:Label, euvoc:XlNote etc.).</p>
+    <p>This is a beta version of the report and is subject to changes.</p>
+
     <h1>Concepts</h1>
     {% include "sections/added_instance_concepts.html" with context %}
     {% include "sections/deleted_instance_concepts.html" with context %}
-    <h1>Alt label</h1>
-    {% include "sections/added_property_concept_alt_label.html" with context %}
-    {% include "sections/deleted_property_concept_alt_label.html" with context %}
-    <h1>Hidden label</h1>
-    {% include "sections/added_property_concept_hidden_label.html" with context %}
-    {% include "sections/deleted_property_concept_hidden_label.html" with context %}
-    <h1>Concept pref label</h1>
+    <h1>Concept status</h1>
+    {% include "sections/added_property_concept_owl_deprecated.html" with context%}
+    {% include "sections/deleted_property_concept_owl_deprecated.html" with context%}
+    <h1>Version info</h1>
+    {% include "sections/added_property_concept_owl_versioninfo.html" with context%}
+    {% include "sections/deleted_property_concept_owl_versioninfo.html" with context%}
+
+    <h1>Preferred labels</h1>
     {% include "sections/added_property_concept_pref_label.html" with context%}
     {% include "sections/deleted_property_concept_pref_label.html" with context %}
-    <h1>Dc identifier</h1>
+{#    <h1>Reified preferred labels</h1>#}
+    {% include "sections/added_reified_concept_pref_label_literal_form.html" with context%}
+    {% include "sections/deleted_reified_concept_pref_label_literal_form.html" with context%}
+
+    <h1>Alternative labels</h1>
+    {% include "sections/added_property_concept_alt_label.html" with context %}
+    {% include "sections/deleted_property_concept_alt_label.html" with context %}
+{#    <h1>Label literal form</h1>#}
+    {% include "sections/added_reified_concept_alt_label_literal_form.html" with context%}
+    {% include "sections/deleted_reified_concept_alt_label_literal_form.html" with context%}
+
+{#    <h1>Hidden labels</h1>#}
+{#    {% include "sections/added_property_concept_hidden_label.html" with context %}#}
+{#    {% include "sections/deleted_property_concept_hidden_label.html" with context %}#}
+
+
+    <h1>Notations and identifiers</h1>
+    {% include "sections/added_property_concept_notation.html" with context%}
+    {% include "sections/deleted_property_concept_notation.html" with context%}
+
+    {% include "sections/added_reified_concept_notation_value.html" with context%}
+    {% include "sections/deleted_reified_concept_notation_value.html" with context%}
+
     {% include "sections/added_property_concept_dc_identifier.html" with context %}
     {% include "sections/deleted_property_concept_dc_identifier.html" with context %}
-    <h1>Dct identifier</h1>
+{#    <h1>Dct identifier</h1>#}
     {% include "sections/added_property_concept_dct_identifier.html" with context%}
     {% include "sections/deleted_property_concept_dct_identifier.html" with context %}
-    <h1>Notation</h1>
-    {% include "sections/added_property_concept_notation.html" with context%}
-    {% include "sections/deleted_property_concept_notation.html" with context%}
-    <h1>SKOS in scheme</h1>
+{#    <h1>Notation</h1>#}
+{#    <h1>Notation value</h1>#}
+
+    <h1>In scheme belonging</h1>
     {% include "sections/added_property_concept_skos_in_scheme.html" with context%}
     {% include "sections/deleted_property_concept_skos_in_scheme.html" with context%}
-    <h1>Owl version info</h1>
-    {% include "sections/added_property_concept_owl_versioninfo.html" with context%}
-    {% include "sections/deleted_property_concept_owl_versioninfo.html" with context%}
-    <h1>SKOS note</h1>
-    {% include "sections/added_property_concept_skos_note.html" with context%}
-    {% include "sections/deleted_property_concept_skos_note.html" with context%}
-    <h1>SKOS change note</h1>
-    {% include "sections/added_property_concept_skos_changeNote.html" with context%}
-    {% include "sections/deleted_property_concept_skos_changeNote.html" with context%}
-    <h1>SKOS definition</h1>
+
+    <h1>Definitions and scope notes</h1>
+{#    <h1>SKOS definition</h1>#}
     {% include "sections/added_property_concept_skos_definition.html" with context%}
     {% include "sections/deleted_property_concept_skos_definition.html" with context%}
-    <h1>Editorial note</h1>
+{#    <h1>Scope note</h1>#}
+    {% include "sections/added_property_concept_scope_note.html" with context%}
+    {% include "sections/deleted_property_concept_scope_note.html" with context%}
+
+    <h1>Other notes</h1>
+{#    <h1>SKOS change note</h1>#}
+    {% include "sections/added_property_concept_skos_changeNote.html" with context%}
+    {% include "sections/deleted_property_concept_skos_changeNote.html" with context%}
+
+{#    <h1>Editorial notes</h1>#}
     {% include "sections/added_property_concept_editorialNote.html" with context%}
     {% include "sections/deleted_property_concept_editorialNote.html" with context%}
-    <h1>Example</h1>
+{#    <h1>Example</h1>#}
     {% include "sections/deleted_property_concept_example.html" with context%}
     {% include "sections/deleted_property_concept_example.html" with context%}
-    <h1>History note</h1>
+{#    <h1>History note</h1>#}
     {% include "sections/added_property_concept_history_note.html" with context%}
     {% include "sections/deleted_property_concept_history_note.html" with context%}
-    <h1>Scope note</h1>
-    {% include "sections/added_property_concept_scope_note.html" with context%}
-    {% include "sections/deleted_property_concept_scope_note.html" with context%}
-    <h1>OWL deprecated</h1>
-    {% include "sections/added_property_concept_owl_deprecated.html" with context%}
-    {% include "sections/deleted_property_concept_owl_deprecated.html" with context%}
-    <h1>Label literal form</h1>
-    {% include "sections/added_reified_concept_alt_label_literal_form.html" with context%}
-    {% include "sections/deleted_reified_concept_alt_label_literal_form.html" with context%}
-    <h1>Notation value</h1>
-    {% include "sections/added_reified_concept_notation_value.html" with context%}
-    {% include "sections/deleted_reified_concept_notation_value.html" with context%}
-    <h1>Pref label literal form</h1>
-    {% include "sections/added_reified_concept_pref_label_literal_form.html" with context%}
-    {% include "sections/deleted_reified_concept_pref_label_literal_form.html" with context%}
+
+    {% include "sections/added_property_concept_skos_note.html" with context%}
+    {% include "sections/deleted_property_concept_skos_note.html" with context%}
+
 {% endblock %}
diff --git a/templates/diff/templates/sections/added_property_concept_alt_label.html b/templates/diff/templates/sections/added_property_concept_alt_label.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept alt label</h2>
+<h2 class="ui header">Added concept alternative labels</h2>
 <section class="ui basic segment">
-    <p>The table below lists the <strong>added property concept "alt label"</strong>
+    <p>The table below lists the added concept <strong>skos:altLabel</strong> values
     </p>
     {% set select_query =
     "prefix owl: <http://www.w3.org/2002/07/owl#>
@@ -29,7 +29,7 @@ <h2 class="ui header">Added property concept alt label</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue) ?languageTag
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue) ?languageTag
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -80,7 +80,7 @@ <h2 class="ui header">Added property concept alt label</h2>
     {% call mc.render_fetch_results(content, error) %}
         {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
         {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-        {{ mc.pandas_table(content, "Overview") }}
+        {{ mc.pandas_table(content, "Added alternative labels") }}
     {% endcall %}
 
 

diff --git a/templates/diff/templates/sections/added_property_concept_dc_identifier.html b/templates/diff/templates/sections/added_property_concept_dc_identifier.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept dc:identifier</h2>
+<h2 class="ui header">Added concept identifiers (DC)</h2>
 <section class="ui basic segment">
-    <p>The table below lists the property concept <strong>dc:identifier</strong>
+    <p>The table below lists the added concept <strong>dc:identifier</strong> values
     </p>
     {% set select_query =
     "
@@ -30,7 +30,7 @@ <h2 class="ui header">Added property concept dc:identifier</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue)
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue)
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -80,7 +80,7 @@ <h2 class="ui header">Added property concept dc:identifier</h2>
     {% call mc.render_fetch_results(content, error) %}
     {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
     {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-    {{ mc.pandas_table(content, "Overview") }}
+    {{ mc.pandas_table(content, "Added identifiers (DC)") }}
     {% endcall %}
 
 

diff --git a/templates/diff/templates/sections/added_property_concept_dct_identifier.html b/templates/diff/templates/sections/added_property_concept_dct_identifier.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept dct:identifier</h2>
+<h2 class="ui header">Added concept identifiers (DCT)</h2>
 <section class="ui basic segment">
-    <p>The table below lists the property concept <strong>dct:identifier</strong>
+    <p>The table below lists the added concept <strong>dct:identifier</strong> values
     </p>
     {% set select_query =
     "
@@ -30,7 +30,7 @@ <h2 class="ui header">Added property concept dct:identifier</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue)
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue)
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -80,7 +80,7 @@ <h2 class="ui header">Added property concept dct:identifier</h2>
     {% call mc.render_fetch_results(content, error) %}
     {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
     {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-    {{ mc.pandas_table(content, "Overview") }}
+    {{ mc.pandas_table(content, "Added dct:identifier") }}
     {% endcall %}
 
 

diff --git a/templates/diff/templates/sections/added_property_concept_editorialNote.html b/templates/diff/templates/sections/added_property_concept_editorialNote.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept editorial note</h2>
+<h2 class="ui header">Added concept editorial notes</h2>
 <section class="ui basic segment">
-    <p>The table below lists the <strong>added property concept "editorial note"</strong>
+    <p>The table below lists the added concept <strong>skos:editorialNote</strong> values
     </p>
     {% set select_query =
     "prefix owl: <http://www.w3.org/2002/07/owl#> 
@@ -29,7 +29,7 @@ <h2 class="ui header">Added property concept editorial note</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue)
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue)
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -79,7 +79,7 @@ <h2 class="ui header">Added property concept editorial note</h2>
     {% call mc.render_fetch_results(content, error) %}
         {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
         {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-        {{ mc.pandas_table(content, "Overview") }}
+        {{ mc.pandas_table(content, "Addedd editorial notes") }}
     {% endcall %}
 
 

diff --git a/templates/diff/templates/sections/added_property_concept_example.html b/templates/diff/templates/sections/added_property_concept_example.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept example</h2>
+<h2 class="ui header">Added concept example notes</h2>
 <section class="ui basic segment">
-    <p>The table below lists the <strong>added property concept "example"</strong>
+    <p>The table below lists the added concept <strong>skos:example</strong> values
     </p>
     {% set select_query =
     "prefix owl: <http://www.w3.org/2002/07/owl#> 
@@ -29,7 +29,7 @@ <h2 class="ui header">Added property concept example</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue)
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue)
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -79,7 +79,7 @@ <h2 class="ui header">Added property concept example</h2>
     {% call mc.render_fetch_results(content, error) %}
         {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
         {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-        {{ mc.pandas_table(content, "Overview") }}
+        {{ mc.pandas_table(content, "Added example notes") }}
     {% endcall %}
 
 

diff --git a/templates/diff/templates/sections/added_property_concept_hidden_label.html b/templates/diff/templates/sections/added_property_concept_hidden_label.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept hidden label</h2>
+<h2 class="ui header">Added concept hidden labels</h2>
 <section class="ui basic segment">
-    <p>The table below lists the <strong>added property concept scheme "hidden label"</strong>
+    <p>The table below lists the added concept <strong>skos:hiddenLabel</strong> values
     </p>
     {% set select_query =
     "prefix owl: <http://www.w3.org/2002/07/owl#>
@@ -29,7 +29,7 @@ <h2 class="ui header">Added property concept hidden label</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue) ?languageTag
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue) ?languageTag
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {

diff --git a/templates/diff/templates/sections/added_property_concept_history_note.html b/templates/diff/templates/sections/added_property_concept_history_note.html
@@ -1,8 +1,8 @@
 {% import "macros.html" as mc %}
 
-<h2 class="ui header">Added property concept history note</h2>
+<h2 class="ui header">Added concept history note</h2>
 <section class="ui basic segment">
-    <p>The table below lists the <strong>added property concept "history note"</strong>
+    <p>The table below lists the added concept <strong>skos:historyNote</strong> values
     </p>
     {% set select_query =
     "prefix owl: <http://www.w3.org/2002/07/owl#> 
@@ -29,7 +29,7 @@ <h2 class="ui header">Added property concept history note</h2>
 PREFIX sd: <http://www.w3.org/ns/sparql-service-description#>
 PREFIX sh: <http://purl.org/skos-history/>
 PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#>
-SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) ?property (?value AS ?addedValue)
+SELECT DISTINCT ?instance (str(?prefLabel) as ?instanceLabel) (?value AS ?addedValue)
 WHERE {
   GRAPH ?versionHistoryGraph {
     VALUES ( ?versionHistoryGraph ?oldVersion ?newVersion ?class ?property ?language) {
@@ -79,7 +79,7 @@ <h2 class="ui header">Added property concept history note</h2>
     {% call mc.render_fetch_results(content, error) %}
         {# TODO: need a function that adds a relative count column, so that we can see the percentage of classes #}
         {# {{ content["percentage"] = content["instances"]  / content["instances"].sum() }}#}
-        {{ mc.pandas_table(content, "Overview") }}
+        {{ mc.pandas_table(content, "Added history notes") }}
     {% endcall %}