Skip to content

Commit

Permalink
parameter settings in prov data
Browse files Browse the repository at this point in the history
  • Loading branch information
@muddymudskipper
muddymudskipper committed Jul 1, 2024
1 parent 7be7764 commit 25e170a
Show file tree
Hide file tree
Showing 11 changed files with 48 additions and 40 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p

## [Unreleased]


### Fixed

- `prov:generatedBy` in output graphs now refers to a plugin IRI instead of a literal

### Changed

- Keep original output ("No explanations found.") if no inconsistencies found with Validate plugin
- Provenance data in output graphs now includes plugin parameter settings


## [1.0.0beta1] 2024-07-01
Expand Down
9 changes: 6 additions & 3 deletions cmem_plugin_reason/plugin_reason.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@
ROBOT,
create_xml_catalog_file,
get_graphs_tree,
post_provenance,
remove_temp,
send_result,
)

PLUGIN_IRI = "https://plugin.eccenca.com/cmem-plugin-reason/reason"


@Plugin(
label="Reason",
Expand Down Expand Up @@ -268,7 +267,6 @@ def reason(self, graphs: dict) -> None:
f"--language-annotation rdfs:comment "
f'"Reasoning result set of <{self.data_graph_iri}> and '
f'<{self.ontology_graph_iri}>" en '
f'--link-annotation prov:wasGeneratedBy "{PLUGIN_IRI}/{self.reasoner}" '
f'--link-annotation prov:wasDerivedFrom "{self.data_graph_iri}" '
f"--link-annotation prov:wasDerivedFrom "
f'"{self.ontology_graph_iri}" '
Expand All @@ -292,4 +290,9 @@ def execute(self, inputs: tuple, context: ExecutionContext) -> None: # noqa: AR
self.reason(graphs)
setup_cmempy_user_access(context.user)
send_result(self.result_graph_iri, Path(self.temp) / "result.ttl")
post_provenance(
self.result_graph_iri,
"cmem_plugin_reason-plugin_reason-ReasonPlugin",
context,
)
remove_temp(self)
29 changes: 7 additions & 22 deletions cmem_plugin_reason/plugin_validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,11 @@
ROBOT,
create_xml_catalog_file,
get_graphs_tree,
post_provenance,
remove_temp,
send_result,
)

PLUGIN_IRI = "https://plugin.eccenca.com/cmem-plugin-reason/validate"


@Plugin(
label="Validate",
Expand Down Expand Up @@ -152,7 +151,6 @@ def validate(self, graphs: dict) -> None:
f'--language-annotation rdfs:label "Ontology Validation Result {utctime}" en '
f"--language-annotation rdfs:comment "
f'"Ontology validation of <{self.ontology_graph_iri}>" en '
f'--link-annotation prov:wasGeneratedBy "{PLUGIN_IRI}/{self.reasoner}" '
f'--link-annotation prov:wasDerivedFrom "{self.ontology_graph_iri}" '
f'--typed-annotation dc:created "{utctime}" xsd:dateTime '
f'--output "{self.temp}/output.ttl"'
Expand All @@ -175,25 +173,6 @@ def make_resource(self, context: ExecutionContext) -> None:
replace=True,
)

def post_provenance(self, plugin_id: str, context: ExecutionContext) -> None:
"""TODO: Post provenance with query"""
plugin_iri = f"http://dataintegration.eccenca.com/{context.task.project_id()}/{context.task.task_id()}"
project_graph = f"http://di.eccenca.com/project/{context.task.project_id()}"
construct = f"""
PREFIX dif: <https://vocab.eccenca.com/di/functions/>
CONSTRUCT {{
GRAPH <{self.output_graph_iri}> {{
<{self.output_graph_iri}> prov:generatedBy <{plugin_iri}> .
<{plugin_iri}> a <https://vocab.eccenca.com/di/functions/{plugin_id}> .
<{plugin_iri}> ?p ?o .
}}
}}
FROM <{project_graph}>
WHERE {{
<{plugin_iri}> ?p ?o .
FILTER((STRSTARTS(STR(?p), 'https://vocab.eccenca.com/di/functions/param_'))
}}""" # noqa: F841

def execute(self, inputs: tuple, context: ExecutionContext) -> Entities | None: # noqa: ARG002
"""Run the workflow operator."""
setup_cmempy_user_access(context.user)
Expand All @@ -205,6 +184,12 @@ def execute(self, inputs: tuple, context: ExecutionContext) -> Entities | None:
if self.produce_graph:
setup_cmempy_user_access(context.user)
send_result(self.output_graph_iri, Path(self.temp) / "output.ttl")
post_provenance(
self.output_graph_iri,
"cmem_plugin_reason-plugin_validate-ValidatePlugin",
context,
)

if self.write_md:
setup_cmempy_user_access(context.user)
self.make_resource(context)
Expand Down
27 changes: 26 additions & 1 deletion cmem_plugin_reason/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
from xml.etree.ElementTree import Element, SubElement, tostring

from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, post_streamed
from cmem.cmempy.dp.proxy.update import post
from cmem_plugin_base.dataintegration.description import PluginParameter
from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType
from cmem_plugin_base.dataintegration.parameter.graph import GraphParameterType
from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
from cmem_plugin_base.dataintegration.plugins import ExecutionContext, WorkflowPlugin
from cmem_plugin_base.dataintegration.types import IntParameterType
from defusedxml import minidom

Expand Down Expand Up @@ -115,3 +116,27 @@ def remove_temp(plugin: WorkflowPlugin) -> None:
rmtree(plugin.temp)
except (OSError, FileNotFoundError) as err:
plugin.log.warning(f"Cannot remove directory {plugin.temp} ({err})")


def post_provenance(graph: str, plugin_id: str, context: ExecutionContext) -> None:
"""Insert provenance"""
plugin_iri = (
f"http://dataintegration.eccenca.com/{context.task.project_id()}/{context.task.task_id()}"
)
project_graph = f"http://di.eccenca.com/project/{context.task.project_id()}"
query = f"""
INSERT {{
GRAPH <{graph}> {{
<{graph}> <http://www.w3.org/ns/prov#wasGeneratedBy> <{plugin_iri}> .
<{plugin_iri}> a <https://vocab.eccenca.com/di/functions/Plugin_{plugin_id}> .
<{plugin_iri}> ?p ?o .
}}
}}
WHERE {{
GRAPH <{project_graph}> {{
<{plugin_iri}> ?p ?o .
FILTER(STRSTARTS(STR(?p), "https://vocab.eccenca.com/di/functions/param_"))
}}
}}"""

post(query=query)
3 changes: 1 addition & 2 deletions tests/test_elk.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/elk> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down
3 changes: 1 addition & 2 deletions tests/test_emr.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/emr> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down
3 changes: 1 addition & 2 deletions tests/test_hermit.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/hermit> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down
3 changes: 1 addition & 2 deletions tests/test_jfact.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/jfact> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down
3 changes: 1 addition & 2 deletions tests/test_structural.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/structural> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down
3 changes: 1 addition & 2 deletions tests/test_validate_output.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@

<https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/output/> a owl:Ontology ;
rdfs:comment "Ontology validation of <https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom <https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/validate/elk> .
prov:wasDerivedFrom <https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/vocab/> .

<https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/vocab/D_6> a owl:NamedIndividual,
<https://ns.eccenca.com/validateontology/e02aaed014c94e0c91bf960fed127750/vocab/A> .
Expand Down
3 changes: 1 addition & 2 deletions tests/test_whelk.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
owl:imports vocab: ;
rdfs:comment "Reasoning result set of <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> and <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/>"@en ;
prov:wasDerivedFrom
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> ;
prov:wasGeneratedBy <https://plugin.eccenca.com/cmem-plugin-reason/reason/whelk> .
<https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/data/> , <https://ns.eccenca.com/reasoning/e02aaed014c94e0c91bf960fed127750/vocab/> .

#################################################################
# Individuals
Expand Down

0 comments on commit 25e170a

Please sign in to comment.