Added toml generalizer (#90)

* Added toml generalizer

* modifying test

Author: cmungall

poetry.lock

@@ -36,6 +36,7 @@ pypandoc = "^1.7.5"
 twine = "^4.0.0"
 oaklib = "^0.1.17"
 pandera = "^0.11.0"
+tomlkit = "^0.11.4"
 
 
 [tool.poetry.dev-dependencies]

pyproject.toml

@@ -237,6 +237,31 @@ def generalize_json(input, output, schema_name, format, omit_null, **kwargs):
     write_schema(schema, output)
 
 
+@main.command()
+@click.argument('input')
+@output_option
+@schema_name_option
+@click.option('--container-class-name', default='Container', help="name of root class")
+@click.option('--enum-columns', '-E', multiple=True, help='column(s) that is forced to be an enum')
+@click.option('--enum-mask-columns', multiple=True, help='column(s) that are excluded from being enums')
+@click.option('--max-enum-size', default=50, help='do not create an enum if more than max distinct members')
+@click.option('--enum-threshold', default=0.1, help='if the number of distinct values / rows is less than this, do not make an enum')
+@click.option('--omit-null/--no-omit-null', default=False, help="if true, ignore null values")
+def generalize_toml(input, output, schema_name, omit_null, **kwargs):
+    """
+    Generalizes from a TOML file to a schema
+
+    See :ref:`generalizers` for more on the generalization framework
+
+    Example:
+
+        schemauto generalize-toml my/data/conf.toml -o my.yaml
+    """
+    ie = JsonDataGeneralizer(omit_null=omit_null)
+    schema = ie.convert(input, format='toml', **kwargs)
+    write_schema(schema, output)
+
+
 @main.command()
 @click.argument('input')
 @output_option

schema_automator/cli.py

@@ -2,6 +2,8 @@
 from typing import Union, Dict, List, Any
 from collections import defaultdict
 import json
+
+import tomlkit
 import yaml
 import gzip
 
@@ -53,6 +55,11 @@ def convert(self, input: Union[str, Dict], format: str = 'json',
                     obj = json.load(stream)
                 elif format == 'yaml':
                     obj = yaml.safe_load(stream)
+                elif format == 'toml':
+                    obj_str = "".join(stream.readlines())
+                    toml_obj = tomlkit.parse(obj_str)
+                    json_str = json.dumps(toml_obj)
+                    obj = json.loads(json_str)
                 else:
                     raise Exception(f'bad format {format}')
         rows_by_table = defaultdict(list)

schema_automator/generalizers/json_instance_generalizer.py

@@ -0,0 +1,39 @@
+import click
+from typing import Union, Dict, List, Any
+from collections import defaultdict
+import json
+import yaml
+import gzip
+
+from dataclasses import dataclass
+
+from linkml_runtime import SchemaView
+from linkml_runtime.linkml_model import SchemaDefinition
+
+from schema_automator import JsonDataGeneralizer
+from schema_automator.generalizers.generalizer import Generalizer
+from schema_automator.generalizers.csv_data_generalizer import CsvDataGeneralizer
+from linkml_runtime.utils.formatutils import camelcase
+
+from schema_automator.utils.schemautils import write_schema
+
+
+@dataclass
+class TomlDataGeneralizer(Generalizer):
+    """
+    A generalizer that abstract from TOML instance data
+    """
+    mappings: dict = None
+    omit_null: bool = None
+
+    def convert(self, input: str, **kwargs) -> SchemaDefinition:
+        """
+        Generalizes from a JSON file
+
+        :param input:
+        :param kwargs:
+        :return:
+        """
+        w
+        json_engine = JsonDataGeneralizer(**kwargs)
+        return json_engine.convert(input, format='toml', **kwargs)

schema_automator/generalizers/toml_instance_generalizer.py

@@ -0,0 +1,53 @@
+# Chemical derivatization method for global GC-MS metabolomics  
+title = 'MPLEx extraction (Grass) '
+author = 'First Last'
+section = 'Metabolomics'
+updated = '08/10/2022'
+protocol_url = ''
+protocol_doi = ''
+description = 'Procedure for chloroform/methanol Folch-type extraction applicable for soil and environmental samples.'
+
+[sample-storage-activity]
+id = 'mplex:1'
+organic_matter = { numeric_value = '1', unit = 'g' }
+contanier = { type = 'Olympus', numeric_value = '50', unit = 'mL'}
+temperature = { numeric_value = '-70', unit = 'C' }
+
+[homogenization-activity]
+id = 'mplex:2'
+source_material = 'mplex:1'
+solvent = 'MeOH/Water'
+concentration = { numeric_value = '4/3', unit = 'ratio' }
+volume = { numeric_value = '15', unit = 'mL' }
+shaker = {type = 'vortex'}
+instrument = {type = 'Omni Homogenizer', details = "disposable probes"}
+
+[solubilization-activity]
+id = 'mplex:3'
+source_material = 'mplex:2'
+solvent = {type = 'Chloroform', temperature = "ice-cold"}
+volume = { numeric_value = '18.5', unit = 'mL' }
+shaker = {type = 'vortex', numeric_value = '1', unit = 'minutes'}
+
+[sample-handling-activity]
+id = 'mplex:4'
+source_material = 'mplex:3'
+activity = {type = 'ice', numeric_value = '5', unit = 'minutes'}
+
+[mixing-activity]
+id = 'mplex:5'
+source_material = 'mplex:4'
+shaker = {type = 'vortex', numeric_value = '1', unit = 'minutes'}
+
+[centrifugation-activity]
+id = 'mplex:6'
+source_material = 'mplex:5'
+centrifuge = {speed = '5000', numeric_value = '10', unit = 'minutes'}
+temperature = { numeric_value = '4', unit = 'C' }
+
+[transfer-activity]
+id = 'derive:12'
+source_material = 'derive:11'
+type = 'supernatant'
+
+# To derivatization (Optional)