From 8d692e4d136b80a390f2921ceabe5deeb0fbf0ab Mon Sep 17 00:00:00 2001 From: muddymudskipper Date: Fri, 28 Jun 2024 09:52:32 +0100 Subject: [PATCH] add max ram percentage parameter, axiom parameters not advanced, move common parameters to utils --- cmem_plugin_reason/plugin_reason.py | 46 ++++++++++----------------- cmem_plugin_reason/plugin_validate.py | 30 ++++++++--------- cmem_plugin_reason/utils.py | 29 +++++++++++++++++ 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/cmem_plugin_reason/plugin_reason.py b/cmem_plugin_reason/plugin_reason.py index 331464d..f327299 100644 --- a/cmem_plugin_reason/plugin_reason.py +++ b/cmem_plugin_reason/plugin_reason.py @@ -13,13 +13,18 @@ from cmem_plugin_base.dataintegration.context import ExecutionContext from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter from cmem_plugin_base.dataintegration.entity import Entities -from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType from cmem_plugin_base.dataintegration.parameter.graph import GraphParameterType from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin -from cmem_plugin_base.dataintegration.types import BoolParameterType, StringParameterType +from cmem_plugin_base.dataintegration.types import ( + BoolParameterType, + StringParameterType, +) from cmem_plugin_base.dataintegration.utils import setup_cmempy_user_access from cmem_plugin_reason.utils import ( + MAX_RAM_PERCENTAGE_PARAMETER, + ONTOLOGY_GRAPH_IRI_PARAMETER, + REASONER_PARAMETER, REASONERS, ROBOT, create_xml_catalog_file, @@ -37,6 +42,9 @@ to a specified graph. The following reasoner options are supported: ELK, Expression Materializing Reasoner, HermiT, JFact, Structural Reasoner and Whelk.""", parameters=[ + REASONER_PARAMETER, + ONTOLOGY_GRAPH_IRI_PARAMETER, + MAX_RAM_PERCENTAGE_PARAMETER, PluginParameter( param_type=GraphParameterType( classes=[ @@ -49,12 +57,6 @@ label="Data graph IRI", description="The IRI of the input data graph.", ), - PluginParameter( - param_type=GraphParameterType(classes=["http://www.w3.org/2002/07/owl#Ontology"]), - name="ontology_graph_iri", - label="Ontology_graph_IRI", - description="The IRI of the input ontology graph.", - ), PluginParameter( param_type=StringParameterType(), name="result_graph_iri", @@ -62,20 +64,12 @@ description="The IRI of the output graph for the reasoning result. " "WARNING: existing graph will be overwritten!", ), - PluginParameter( - param_type=ChoiceParameterType(REASONERS), - name="reasoner", - label="Reasoner", - description="Reasoner option.", - default_value="elk", - ), PluginParameter( param_type=BoolParameterType(), name="sub_class", label="SubClass", description="", default_value=True, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -83,7 +77,6 @@ label="EquivalentClass", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -91,7 +84,6 @@ label="DisjointClasses", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -99,7 +91,6 @@ label="DataPropertyCharacteristic", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -107,7 +98,6 @@ label="EquivalentDataProperties", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -115,7 +105,6 @@ label="SubDataProperty", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -123,7 +112,6 @@ label="ClassAssertion", description="Generated Axioms", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -131,7 +119,6 @@ label="PropertyAssertion", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -139,7 +126,6 @@ label="EquivalentObjectProperty", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -147,7 +133,6 @@ label="InverseObjectProperties", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -155,7 +140,6 @@ label="ObjectPropertyCharacteristic", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -163,7 +147,6 @@ label="SubObjectProperty", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -171,7 +154,6 @@ label="ObjectPropertyRange", description="", default_value=False, - advanced=True, ), PluginParameter( param_type=BoolParameterType(), @@ -179,7 +161,6 @@ label="ObjectPropertyDomain", description="", default_value=False, - advanced=True, ), ], ) @@ -206,6 +187,7 @@ def __init__( # noqa: PLR0913 sub_class: bool = True, sub_data_property: bool = False, sub_object_property: bool = False, + max_ram_percentage: int = 15, ) -> None: """Init""" self.axioms = { @@ -242,6 +224,8 @@ def __init__( # noqa: PLR0913 errors += "Invalid value for parameter Reasoner. " if True not in self.axioms.values(): errors += "No axiom generator selected. " + if max_ram_percentage not in range(1, 100): + errors += "Invalid value for parameter Maximum RAM Percentage. " if errors: raise ValueError(errors[:-1]) @@ -249,6 +233,7 @@ def __init__( # noqa: PLR0913 self.ontology_graph_iri = ontology_graph_iri self.result_graph_iri = result_graph_iri self.reasoner = reasoner + self.max_ram_percentage = max_ram_percentage self.temp = f"reason_{uuid4().hex}" def get_graphs(self, graphs: dict, context: ExecutionContext) -> None: @@ -271,7 +256,8 @@ def reason(self, graphs: dict) -> None: data_location = f"{self.temp}/{graphs[self.data_graph_iri]}" utctime = str(datetime.fromtimestamp(int(time()), tz=UTC))[:-6].replace(" ", "T") + "Z" cmd = ( - f'java -XX:MaxRAMPercentage=15 -jar {ROBOT} merge --input "{data_location}" ' + f"java -XX:MaxRAMPercentage={self.max_ram_percentage} -jar {ROBOT} " + f'merge --input "{data_location}" ' "--collapse-import-closure false " f"reason --reasoner {self.reasoner} " f'--axiom-generators "{axioms}" ' diff --git a/cmem_plugin_reason/plugin_validate.py b/cmem_plugin_reason/plugin_validate.py index 0e7462f..b3bae29 100644 --- a/cmem_plugin_reason/plugin_validate.py +++ b/cmem_plugin_reason/plugin_validate.py @@ -19,14 +19,15 @@ EntityPath, EntitySchema, ) -from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType -from cmem_plugin_base.dataintegration.parameter.graph import GraphParameterType from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin from cmem_plugin_base.dataintegration.types import BoolParameterType, StringParameterType from cmem_plugin_base.dataintegration.utils import setup_cmempy_user_access from pathvalidate import validate_filename from cmem_plugin_reason.utils import ( + MAX_RAM_PERCENTAGE_PARAMETER, + ONTOLOGY_GRAPH_IRI_PARAMETER, + REASONER_PARAMETER, REASONERS, ROBOT, create_xml_catalog_file, @@ -41,19 +42,9 @@ documentation="""""", icon=Icon(package=__package__, file_name="obofoundry.png"), parameters=[ - PluginParameter( - param_type=GraphParameterType(classes=["http://www.w3.org/2002/07/owl#Ontology"]), - name="ontology_graph_iri", - label="Ontology_graph_IRI", - description="The IRI of the input ontology graph.", - ), - PluginParameter( - param_type=ChoiceParameterType(REASONERS), - name="reasoner", - label="Reasoner", - description="Reasoner option.", - default_value="elk", - ), + REASONER_PARAMETER, + ONTOLOGY_GRAPH_IRI_PARAMETER, + MAX_RAM_PERCENTAGE_PARAMETER, PluginParameter( param_type=BoolParameterType(), name="write_md", @@ -103,6 +94,7 @@ def __init__( # noqa: PLR0913 write_md: bool = False, md_filename: str = "", stop_at_inconsistencies: bool = False, + max_ram_percentage: int = 15, ) -> None: errors = "" if not validators.url(ontology_graph_iri): @@ -116,6 +108,8 @@ def __init__( # noqa: PLR0913 validate_filename(md_filename) except: # noqa: E722 errors += "Invalid filename for parameter Output filename. " + if max_ram_percentage not in range(1, 100): + errors += "Invalid value for parameter Maximum RAM Percentage. " if errors: raise ValueError(errors[:-1]) @@ -125,8 +119,9 @@ def __init__( # noqa: PLR0913 self.output_graph_iri = output_graph_iri self.write_md = write_md self.stop_at_inconsistencies = stop_at_inconsistencies - self.temp = f"reason_{uuid4().hex}" self.md_filename = md_filename if md_filename and write_md else "mdfile.md" + self.max_ram_percentage = max_ram_percentage + self.temp = f"reason_{uuid4().hex}" def get_graphs(self, graphs: dict, context: ExecutionContext) -> None: """Get graphs from CMEM""" @@ -143,7 +138,8 @@ def validate(self, graphs: dict) -> None: utctime = str(datetime.fromtimestamp(int(time()), tz=UTC))[:-6].replace(" ", "T") + "Z" cmd = ( - f'java -XX:MaxRAMPercentage=15 -jar {ROBOT} merge --input "{data_location}" ' + f"java -XX:MaxRAMPercentage={self.max_ram_percentage} -jar {ROBOT} " + f'merge --input "{data_location}" ' f"explain --reasoner {self.reasoner} -M inconsistency " f'--explanation "{self.temp}/{self.md_filename}"' ) diff --git a/cmem_plugin_reason/utils.py b/cmem_plugin_reason/utils.py index d7196af..9323edb 100644 --- a/cmem_plugin_reason/utils.py +++ b/cmem_plugin_reason/utils.py @@ -11,6 +11,10 @@ ) from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, post_streamed +from cmem_plugin_base.dataintegration.description import PluginParameter +from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType +from cmem_plugin_base.dataintegration.parameter.graph import GraphParameterType +from cmem_plugin_base.dataintegration.types import IntParameterType from defusedxml import minidom from . import __path__ @@ -28,6 +32,31 @@ } ) +ONTOLOGY_GRAPH_IRI_PARAMETER = PluginParameter( + param_type=GraphParameterType(classes=["http://www.w3.org/2002/07/owl#Ontology"]), + name="ontology_graph_iri", + label="Ontology_graph_IRI", + description="The IRI of the input ontology graph.", +) + +REASONER_PARAMETER = PluginParameter( + param_type=ChoiceParameterType(REASONERS), + name="reasoner", + label="Reasoner", + description="Reasoner option.", + default_value="elk", +) + +MAX_RAM_PERCENTAGE_PARAMETER = PluginParameter( + param_type=IntParameterType(), + name="max_ram_percentage", + label="Maximum RAM Percentage", + description="Maximum heap size for the Java virtual machine running the reasoning " + "process. ⚠️ Setting the RAM usage too high may result in an out of memory error.", + default_value=15, + advanced=True, +) + def convert_iri_to_filename(value: str) -> str: """Convert IRI to filename"""