Added pint-preprocessor & helpers for ucum-pint mappings

Author: dalito

README.md

@@ -1,7 +1,8 @@
 # Easier access to UCUM from Python
 
 > **This is almost done. Feedback welcome!**
-The lark grammar to parse UCUM codes and the transformer that converts UCUM units to pint are implemented.
+> The lark grammar to parse UCUM codes and the transformer that converts UCUM units to pint are implemented.
+> For some UCUM units we still have to define pint units or aliases and for some also name mappings.
 
 [UCUM](https://ucum.org/) (Unified Code for Units of Measure) is a code system intended to cover all units of measures.
 It provides a formalism to express units in an unambiguous way suitable for electronic communication.
@@ -18,12 +19,16 @@ So updating the parser for new UCUM releases is straight forward.
 The parser is built with the great [lark](https://pypi.org/project/lark/) parser toolkit.
 The generated lark grammar file for case-sensitive UCUM codes is included in the repository, see [ucum_grammar.lark](https://github.com/dalito/ucumvert/blob/main/src/ucumvert/ucum_grammar.lark).
 
+Some of the UCUM unit atoms are invalid unit names in pint, for example `cal_[15]`, `m[H2O]`, `10*`, `[in_i'H2O]`.
+For all of them we define mappings to valid pint unit names in [ucum_pint.py](https://github.com/dalito/ucumvert/blob/main/src/ucumvert/ucum_pint.py), e.g. `{"cal_[15]": "cal_15"}`.
+
 ## Install
 
 Installation from git in developer mode including creation of virtual environment (pip should be newer than 23.1):
 
 Linux
-```
+
+```bash
 git clone https://github.com/dalito/ucumvert.git
 cd ucumvert
 python -m venv .venv
@@ -32,7 +37,8 @@ pip install -e .[dev]
 ```
 
 Windows
-```
+
+```bash
 git clone https://github.com/dalito/ucumvert.git
 cd ucumvert
 py -m venv .venv
@@ -88,17 +94,17 @@ You may use the package in your code for converting UCUM codes to pint like this
 
 The unit tests include a test to parse all common UCUM unit codes from the official repo. To see this run
 
-```cmd
-$ pytest
+```bash
+pytest
 ```
 
 The common UCUM unit codes are available only in binary form (xlsx, docs, pdf).
 Here we keep a copy in tsv-format `ucum_examples.tsv`.
 To (re)generate this tsv-file from the official xlsx-file in the [UCUM repository](https://github.com/ucum-org/ucum/tree/main/common-units) run
 
-```cmd
-$ pip install openpyxl
-$ python src/src/ucumvert/vendor/get_ucum_example_as_tsv.py
+```bash
+pip install openpyxl
+python src/src/ucumvert/vendor/get_ucum_example_as_tsv.py
 ```
 
 ## Useful links

src/ucumvert/__init__.py

@@ -3,7 +3,11 @@
     make_parse_tree_png,
     update_lark_ucum_grammar_file,
 )
-from ucumvert.ucum_pint import UcumToPintTransformer
+from ucumvert.ucum_pint import (
+    UcumToPintTransformer,
+    get_pint_registry,
+    ucum_preprocessor,
+)
 
 try:
     from ucumvert._version import __version__, __version_tuple__
@@ -13,7 +17,9 @@
 
 __all__ = [
     "get_ucum_parser",
+    "get_pint_registry",
     "make_parse_tree_png",
+    "ucum_preprocessor",
     "update_lark_ucum_grammar_file",
     "UcumToPintTransformer",
 ]

src/ucumvert/ucum_pint.py

@@ -2,26 +2,41 @@
 
 import pint
 from lark import Transformer
+from lark.exceptions import VisitError
 
 from ucumvert.parser import (
     get_ucum_parser,
     make_parse_tree_png,
     update_lark_ucum_grammar_file,
 )
+from ucumvert.xml_util import get_metric_units, get_non_metric_units, get_prefixes
 
 # Some UCUM unit atoms are syntactically incompatiple with pint. For these we
 # map to a pint-compatible unit name which we define in pint_ucum_defs.txt
-# as alias or new unit.
-# TODO Define the commented out units in pint_ucum_defs.txt
+# as alias or new unit. To determine what needs a mapping, use the function
+# "find_ucum_codes_that_need_mapping()" below.
 
-mappings_ucum_to_pint = {
+MAPPINGS_UCUM_TO_PINT = {
     # "UCUM_unit_atom": "pint_unit_name_or_alias"
+    # === prefixes ===
+    # all good!
     # === metric units ===
     "cal_[20]": "cal_20",
     "cal_[15]": "cal_15",
+    # m[H2O]
+    # m[Hg]
+    "g%": "g%",  # invalid as unit name but correctly parsed as <Unit('gram * percent')>
+    # B[SPL]
+    # B[V]
+    # B[mV]
+    # B[uV]
+    # B[10.nV]
+    # B[W]
+    # B[kW]
     # === non-metric units ===
     "10*": "_10",
     "10^": "_10",
+    "%": "%",  # invalid as unit name but correctly parsed as <Unit('percent')>
     "'": "minute",
     "''": "second",
     "[in_i'H2O]": "in_i_H2O",
@@ -58,13 +73,13 @@
 class UcumToPintTransformer(Transformer):
     def __init__(self, ureg=None):
         if ureg is None:
-            self.ureg = pint.UnitRegistry()
-            # Append the local definitions for ucum units to the default registry
-            self.ureg.load_definitions(
-                Path(__file__).resolve().parent / "pint_ucum_defs.txt"
-            )
+            self.ureg = pint.UnitRegistry(on_redefinition="raise")
         else:
             self.ureg = ureg
+        # Append the local definitions for ucum units to the default registry
+        self.ureg.load_definitions(
+            Path(__file__).resolve().parent / "pint_ucum_defs.txt"
+        )
 
     def main_term(self, args):
         # print("DBGmt>", repr(args), len(args))
@@ -104,7 +119,7 @@ def simple_unit(self, args):
             return self.ureg(args[0] + args[1])
 
         # Substitute UCUM atoms that cannot be defined in pint as units or aliases.
-        return self.ureg(mappings_ucum_to_pint.get(args[0], args[0]))
+        return self.ureg(MAPPINGS_UCUM_TO_PINT.get(args[0], args[0]))
 
     def annotatable(self, args):
         # print("DBGan>", repr(args), len(args))
@@ -113,6 +128,133 @@ def annotatable(self, args):
         return args[0]
 
 
+class UcumToPintStrTransformer(Transformer):
+    def main_term(self, args):
+        # print("DBGmt>", repr(args), len(args))
+        if len(args) == 2:  # unary DIVIDE  # noqa: PLR2004
+            if getattr(args[1], "type", None):  # no unit, only an ANNOTATION
+                return "1"  # will create <Quantity(1, 'dimensionless')>
+            return f"(1 / {args[1]})"
+        return f"({args[0]})"
+
+    def term(self, args):
+        # print("DBGt>", repr(args), len(args))
+        if len(args) == 3:  # noqa: PLR2004
+            if (
+                getattr(args[0], "type", None) == "ANNOTATION"
+            ):  # first term is annotation
+                args[0] = "1"
+            if (
+                getattr(args[2], "type", None) == "ANNOTATION"
+            ):  # second term is annotation
+                args[2] = "1"
+            if args[1] == ".":  # multiplication
+                return f"({args[0]} * {args[2]})"
+            # division
+            return f"({args[0]} / {args[2]})"
+        return f"({args[0]})"  # no operator, return single component
+
+    def component(self, args):
+        # print("DBGc>", repr(args), len(args))
+        if args[1].type == "ANNOTATION":  # ignore annotations
+            # print(f"dropping annotation: {args[1]}")
+            return f"({args[0]})"
+        return args[:]
+
+    def simple_unit(self, args):
+        # print("DBGsu>", repr(args), len(args))
+        if len(args) == 2:  # prefix is present  # noqa: PLR2004
+            return f"({args[0]} + {args[1]})"
+
+        # Substitute UCUM atoms that cannot be defined in pint as units or aliases.
+        ret = MAPPINGS_UCUM_TO_PINT.get(args[0], args[0])
+        return f"({ret})"
+
+    def annotatable(self, args):
+        # print("DBGan>", repr(args), len(args))
+        if len(args) == 2:  # exponent is present  # noqa: PLR2004
+            return f"{args[0]}**{int(args[1])}"
+        return f"({args[0]})"
+
+
+def ucum_preprocessor(unit_input):
+    """Preprocess UCUM code before parsing as pint unit.
+
+    Usage:
+        ureg = pint.UnitRegistry()
+        ureg.preprocessors.append(ucum_preprocessor)
+    """
+    ucum_parser = get_ucum_parser()
+    transformer = UcumToPintStrTransformer()
+    # print("DBGpp in >", repr(unit_input))
+    parsed_data = ucum_parser.parse(unit_input)
+    # pintified_str = str(transformer.transform(parsed_data))
+    # print(f"DBGpp out> {pintified_str}")
+    return str(transformer.transform(parsed_data))
+
+
+def find_ucum_codes_that_need_mapping(existing_mappings=MAPPINGS_UCUM_TO_PINT):
+    """Find UCUM atoms that are syntactically incompatiple with pint."""
+    print("The following UCUM atoms must be mapped to valid pint unit names.")
+    ureg = pint.UnitRegistry()
+    sections = {
+        "prefixes": get_prefixes,
+        "metric": get_metric_units,
+        "non-metric": get_non_metric_units,
+    }
+    need_mappings = {k: [] for k in sections}
+    for section, get_fcn in sections.items():
+        print(f"\n=== {section} ===")
+        for ucum_code in get_fcn():
+            if ucum_code in existing_mappings:
+                continue
+            def_str = (
+                f"{ucum_code}- = 1" if section == "prefixes" else f"{ucum_code} = 1"
+            )
+            try:
+                ureg.define(def_str)
+            except pint.DefinitionSyntaxError:
+                need_mappings[section].append(ucum_code)
+                print(f"{ucum_code}")
+                continue
+        if not need_mappings[section]:
+            print("all good!")
+    return need_mappings
+
+
+def find_matching_pint_definitions(ureg=None):
+    """Find Pint units that match UCUM units."""
+    if ureg is None:
+        ureg = pint.UnitRegistry()
+    sections = {
+        "prefixes": get_prefixes,
+        "metric": get_metric_units,
+        "non-metric": get_non_metric_units,
+    }
+    ucum_parser = get_ucum_parser()
+    transformer = UcumToPintTransformer(ureg=ureg)
+    for section, get_fcn in sections.items():
+        print(f"\n=== {section} ===")
+        for ucum_code in get_fcn():
+            lookup_str = f"{ucum_code} m" if section == "prefixes" else ucum_code
+            try:
+                parsed_data = ucum_parser.parse(lookup_str)
+            except VisitError as exc:
+                print(f"PARSER ERROR: {exc.args[0]}")
+                raise
+            try:
+                pint_quantity = transformer.transform(parsed_data)
+            except pint.UndefinedUnitError as exc:
+                msg = getattr(exc, "msg", "")
+                print(f"NOT DEFINED: {msg}")
+                continue
+            except VisitError as exc:
+                msg = exc.args[0].splitlines()[-1]
+                print(f"TRANSFORM ERROR: {msg}")
+                continue
+            print(f"{ucum_code} --> {pint_quantity!r}")
+
+
 def run_examples():
     test_ucum_units = [
         # "Cel",
@@ -129,6 +271,21 @@ def run_examples():
         print(f"Pint {q!r}")
 
 
+def get_pint_registry(ureg=None):
+    """Return a pint registry with the UCUM definitions loaded."""
+    if ureg is None:
+        ureg = pint.UnitRegistry(on_redefinition="raise")
+    ureg.preprocessors.append(ucum_preprocessor)
+    ureg.load_definitions(Path(__file__).resolve().parent / "pint_ucum_defs.txt")
+    return ureg
+
+
 if __name__ == "__main__":
     update_lark_ucum_grammar_file()
-    run_examples()
+    # run_examples()
+    # find_ucum_codes_that_need_mapping()
+    # find_matching_pint_definitions()
+
+    # ureg = get_pint_registry()
+    # print(ureg("Cel"))
+    # print(ureg("'"))

tests/conftest.py

@@ -11,7 +11,14 @@ def ucum_parser():
 
 
 @pytest.fixture(scope="session")
-def transform():
+def ureg_std():
+    import pint
+
+    return pint.UnitRegistry()
+
+
+@pytest.fixture(scope="session")
+def transform(ureg_std):
     from ucumvert import UcumToPintTransformer
 
-    return UcumToPintTransformer().transform
+    return UcumToPintTransformer(ureg_std).transform

tests/test_ucum_pint.py

@@ -2,6 +2,7 @@
 from pint import UnitRegistry
 from test_parser import ucum_examples_valid
 from ucumvert import UcumToPintTransformer
+from ucumvert.ucum_pint import find_ucum_codes_that_need_mapping
 from ucumvert.xml_util import get_metric_units, get_non_metric_units
 
 ureg = UnitRegistry()
@@ -12,10 +13,17 @@ def get_unit_atoms():
     return get_metric_units() + get_non_metric_units()
 
 
-def test_ucum_to_pint(ucum_parser):
+def test_find_ucum_codes_that_need_mapping():
+    mappings = find_ucum_codes_that_need_mapping(existing_mappings={})
+    assert len(mappings["prefixes"]) == 0
+    assert len(mappings["metric"]) == 12  # noqa: PLR2004
+    assert len(mappings["non-metric"]) == 33  # noqa: PLR2004
+
+
+def test_ucum_to_pint(ucum_parser, ureg_std):
     expected_quantity = ureg("millimeter")
     parsed_data = ucum_parser.parse("mm")
-    result = UcumToPintTransformer().transform(parsed_data)
+    result = UcumToPintTransformer(ureg=ureg_std).transform(parsed_data)
     assert result == expected_quantity