Merge pull request #93 from FAIRmat-NFDI/helper-update

Update helper functions and streamline formatting

Author: lukaspie

scripts/generate_ref_comments_jsons.py

@@ -1,8 +1,3 @@
-"""
-Parser for reading XPS (X-ray Photoelectron Spectroscopy) data from
-Phi PHI VersaProbe 4 instruments (.spe or .pro format), to be passed to
-mpes nxdl (NeXus Definition Language) template.
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -20,7 +15,9 @@
 # limitations under the License.
 #
 
-# pylint: disable=too-many-lines,too-many-instance-attributes
+"""
+Regenerate reference JSON files for testing the VAMAS comment extraction.
+"""
 
 import os
 import json
@@ -66,7 +63,7 @@ def generate_ref_comment_jsons():
         comments = handle_comments(comment_lines, comment_type=comment_type)
 
         for key, val in comments.items():
-            if type(val) == np.ndarray:
+            if isinstance(val, np.ndarray):
                 comments[key] = val.tolist()
 
         with open(ref_json_filepath, "w") as json_file:

scripts/regenerate_ref_comments_jsons.sh

@@ -1,6 +1,3 @@
-"""
-Data model for Kratos spectrometers for export to CasaXPS.
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -17,10 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# pylint: disable=too-many-instance-attributes
+"""
+Data model for Kratos spectrometers for export to CasaXPS.
+"""
 
-from dataclasses import dataclass
-from dataclasses import field
+from dataclasses import dataclass, field
 
 from pynxtools_xps.reader_utils import XpsDataclass
 

src/pynxtools_xps/kratos/kratos_data_model.py

@@ -1,8 +1,3 @@
-"""
-Parser for reading XPS (X-ray Photoelectron Spectroscopy) metadata from
-Kratos instruments (currently only after exporting to .vms format), to be passed to
-mpes nxdl (NeXus Definition Language) template.
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -19,21 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-# pylint: disable=too-many-lines,too-many-instance-attributes
+"""
+Parser for reading XPS (X-ray Photoelectron Spectroscopy) metadata from
+Kratos instruments (currently only after exporting to .vms format), to be
+passed to MPES nxdl (NeXus Definition Language) template.
+"""
 
 import re
-import datetime
 from typing import Any, Dict, List, Union, Tuple
 from pathlib import Path
 
 from pynxtools_xps.reader_utils import (
     convert_pascal_to_snake,
+    extract_unit,
 )
 
 from pynxtools_xps.value_mappers import (
     convert_bool,
-    convert_units,
+    parse_datetime,
 )
 
 from pynxtools_xps.kratos.kratos_data_model import (
@@ -72,6 +70,8 @@
     "sample_tilt": "degree",
 }
 
+POSSIBLE_DATE_FORMATS: List[str] = ["%d.%m.%Y %H:%M", "%d/%m/%Y %H:%M"]
+
 
 class KratosParser:
     """
@@ -89,7 +89,7 @@ def __init__(self):
         self.metadata = KratosMetadata()
 
         self.value_function_map: Dict[str, Any] = {
-            "date_created": _parse_datetime,
+            "date_created": parse_datetime,
             "description": _convert_description,
             "charge_neutraliser": convert_bool,
             "deflection": _convert_xray_deflection,
@@ -150,7 +150,7 @@ def parse_header_into_metadata(self, header: List[str]):
                 field_type = type(getattr(self.metadata, key))
 
                 if key in KEYS_WITH_UNITS:
-                    value, unit = self.extract_unit(key, value)
+                    value, unit = extract_unit(key, value, UNIT_MISSING)  # type: ignore[assignment]
                     setattr(self.metadata, f"{key}_units", unit)
 
                 value = self.map_values(key, value, field_type)
@@ -159,46 +159,6 @@ def parse_header_into_metadata(self, header: List[str]):
 
         self.metadata.validate_types()
 
-    def extract_unit(self, key: str, value: str) -> Tuple[Any, str]:
-        """
-        Extract units for the metadata containing unit information.
-
-        Example:
-            analyser_work_function: 4.506eV
-            -> analyser_work_function: 4.506,
-               analyser_work_function_units: eV,
-
-        Parameters
-        ----------
-        key : str
-            Key of the associated value.
-        value : str
-            Combined unit and value information.
-
-        Returns
-        -------
-        value :
-            value with units.
-        unit : str
-            Associated unit.
-
-        """
-
-        pattern = re.compile(r"([-+]?\d*\.?\d+(?:[eE][-+]?\d+)?)([a-zA-Z]+)")
-        match = pattern.match(value)
-
-        if match:
-            value, unit = match.groups()
-        else:
-            unit = ""
-
-        unit = convert_units(unit)
-
-        if key in UNIT_MISSING:
-            unit = UNIT_MISSING[key]
-
-        return value, unit
-
     def map_values(self, key: str, value, field_type):
         """
         Map values to corresponding structure and field type.
@@ -221,7 +181,10 @@ def map_values(self, key: str, value, field_type):
 
         if key in self.value_function_map:
             map_fn = self.value_function_map[key]
-            value = map_fn(value)
+            if key == "date_created":
+                value = map_fn(value, POSSIBLE_DATE_FORMATS)
+            else:
+                value = map_fn(value)
         return field_type(value)
 
     def flatten_metadata(self) -> Dict[str, Any]:
@@ -268,34 +231,6 @@ def setup_unit(flattened_dict: Dict[str, Any], unit_key: str):
         return flattened_dict
 
 
-def _parse_datetime(datetime_string: str) -> str:
-    """
-    Convert the native time format to the datetime string
-    in the ISO 8601 format: '%Y-%b-%dT%H:%M:%S.%fZ'.
-
-    Parameters
-    ----------
-    value : str
-        String representation of the date in the format
-        "%Y-%m-%d", "%m/%d/%Y" or "%H:%M:%S", "%I:%M:%S %p".
-
-    Returns
-    -------
-    date_object : str
-        Datetime in ISO8601 format.
-
-    """
-    possible_date_formats = ["%d.%m.%Y %H:%M", "%d/%m/%Y %H:%M"]
-    for date_fmt in possible_date_formats:
-        try:
-            datetime_obj = datetime.datetime.strptime(datetime_string, date_fmt)
-            return datetime_obj.astimezone().isoformat()
-
-        except ValueError:
-            continue
-    raise ValueError("Date and time could not be converted to ISO 8601 format.")
-
-
 def _convert_description(value: str) -> Dict[str, Any]:
     """Map all items in description to a dictionary."""
     pattern = re.compile(

src/pynxtools_xps/kratos/metadata_kratos.py

@@ -1,3 +1,20 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 """
 MKdocs macros for the documentation
 """

src/pynxtools_xps/mkdocs.py

@@ -1,6 +1,3 @@
-"""
-Data model for Phi VersaProbe 4 software (version SS 3.3.3.2).
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -17,10 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# pylint: disable=too-many-instance-attributes
+"""
+Data model for Phi VersaProbe 4 software (version SS 3.3.3.2).
+"""
 
-from dataclasses import dataclass
-from dataclasses import field
+from dataclasses import dataclass, field
 
 import numpy as np
 

src/pynxtools_xps/phi/phi_data_model.py

@@ -1,8 +1,3 @@
-"""
-Parser for reading XPS (X-ray Photoelectron Spectroscopy) data from
-Phi PHI VersaProbe 4 instruments (.spe or .pro format), to be passed to
-mpes nxdl (NeXus Definition Language) template.
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -19,8 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-# pylint: disable=too-many-lines,too-many-instance-attributes
+"""
+Parser for reading XPS (X-ray Photoelectron Spectroscopy) data from
+Phi PHI VersaProbe 4 instruments (.spe or .pro format), to be passed to
+MPES nxdl (NeXus Definition Language) template.
+"""
 
 import re
 import warnings

src/pynxtools_xps/phi/spe_pro_phi.py

@@ -1,7 +1,3 @@
-"""
-A generic reader for loading XPS (X-ray Photoelectron Spectroscopy) data
-file into mpes nxdl (NeXus Definition Language) template.
-"""
 # Copyright The NOMAD Authors.
 #
 # This file is part of NOMAD. See https://nomad-lab.eu for further info.
@@ -19,35 +15,36 @@
 # limitations under the License.
 #
 # pylint: disable=too-many-lines,too-few-public-methods
+"""
+A generic reader for loading XPS (X-ray Photoelectron Spectroscopy) data
+file into mpes nxdl (NeXus Definition Language) template.
+"""
 
-import os
-import sys
-import re
-import datetime
 import copy
+import datetime
 import logging
+import os
+import re
+import sys
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Optional, Union, Set
-import numpy as np
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
+import numpy as np
 from pynxtools.dataconverter.helpers import extract_atom_types
-from pynxtools.dataconverter.readers.multi.reader import (
-    MultiFormatReader,
-)
+from pynxtools.dataconverter.readers.multi.reader import MultiFormatReader
 from pynxtools.dataconverter.readers.utils import parse_yml
 from pynxtools.dataconverter.template import Template
 
+from pynxtools_xps.reader_utils import check_units
+
 from pynxtools_xps.phi.spe_pro_phi import MapperPhi
 from pynxtools_xps.scienta.scienta_reader import MapperScienta
 from pynxtools_xps.specs.sle.sle_specs import SleMapperSpecs
-from pynxtools_xps.specs.xy.xy_specs import XyMapperSpecs
 from pynxtools_xps.specs.xml.xml_specs import XmlMapperSpecs
+from pynxtools_xps.specs.xy.xy_specs import XyMapperSpecs
 from pynxtools_xps.vms.txt_vamas_export import TxtMapperVamasExport
 from pynxtools_xps.vms.vamas import VamasMapper
 
-from pynxtools_xps.reader_utils import check_units
-
-
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)