Merge pull request #175 from khaeru/enh/2024-W16

Miscellaneous improvements for 2024-W16

Author: khaeru

.github/workflows/pytest.yaml

@@ -30,6 +30,14 @@ jobs:
         # commented: only enable once next Python version enters RC
         # - "3.13.0-rc.1"  # Development version
 
+        # Work around https://github.com/actions/setup-python/issues/696
+        exclude:
+        - {os: macos-latest, python-version: "3.8"}
+        - {os: macos-latest, python-version: "3.9"}
+        include:
+        - {os: macos-13, python-version: "3.8"}
+        - {os: macos-13, python-version: "3.9"}
+
       fail-fast: false
 
     runs-on: ${{ matrix.os }}

.pre-commit-config.yaml

@@ -1,21 +1,21 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.9.0
+  rev: v1.10.0
   hooks:
   - id: mypy
     additional_dependencies:
+    - lxml-stubs
     - pandas-stubs
     - pytest
     - requests-cache
     - requests-mock
     - types-Jinja2
-    - types-lxml
     - types-python-dateutil
     - types-PyYAML
     - types-requests
     args: []
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.3
+  rev: v0.4.2
   hooks:
   - id: ruff
   - id: ruff-format

README.rst

@@ -1,21 +1,22 @@
 sdmx: Statistical data and metadata exchange
 ********************************************
+|pypi| |rtd| |services| |gha| |codecov|
 
-.. image:: https://github.com/khaeru/sdmx/actions/workflows/pytest.yaml/badge.svg
-   :target: https://github.com/khaeru/sdmx/actions
-   :alt: Status badge for the "pytest" continuous integration workflow
-.. image:: https://codecov.io/gh/khaeru/sdmx/branch/main/graph/badge.svg
-   :target: https://codecov.io/gh/khaeru/sdmx
-   :alt: Codecov test coverage badge
-.. image:: https://readthedocs.org/projects/sdmx1/badge/?version=latest
-   :target: https://sdmx1.readthedocs.io/en/latest
-   :alt: Documentation status badge
-.. image:: https://img.shields.io/pypi/v/sdmx1.svg
+.. |pypi| image:: https://img.shields.io/pypi/v/sdmx1.svg
    :target: https://pypi.org/project/sdmx1
    :alt: PyPI version badge
-.. image:: https://img.shields.io/badge/services-status-informational
+.. |rtd| image:: https://readthedocs.org/projects/sdmx1/badge/?version=latest
+   :target: https://sdmx1.readthedocs.io/en/latest
+   :alt: Documentation status badge
+.. |services| image:: https://img.shields.io/badge/services-status-informational
    :target: https://khaeru.github.io/sdmx/
    :alt: Status of supported service endpoints
+.. |gha| image:: https://github.com/khaeru/sdmx/actions/workflows/pytest.yaml/badge.svg
+   :target: https://github.com/khaeru/sdmx/actions
+   :alt: Status badge for the "pytest" continuous integration workflow
+.. |codecov| image:: https://codecov.io/gh/khaeru/sdmx/branch/main/graph/badge.svg
+   :target: https://codecov.io/gh/khaeru/sdmx
+   :alt: Codecov test coverage badge
 
 `Source code @ Github <https://github.com/khaeru/sdmx/>`_ —
 `Authors <https://github.com/khaeru/sdmx/graphs/contributors>`_

doc/sources.rst

@@ -235,6 +235,11 @@ In order to identify available data flows:
 SDMX-ML —
 `Website <https://ilostat.ilo.org/resources/sdmx-tools/>`__
 
+.. versionchanged:: 2.15.0
+
+   Sometime before 2024-04-26, the base URL of this source changed from ``https://www.ilo.org/sdmx/rest`` to ``https://sdmx.ilo.org/rest``.
+   The "SDMX query builder" at the above URL reflects the change, but the documentation still shows the old URL, and there does not appear to have been any public announcement about the new URL, retirement of the old URL, etc.
+   Thanks :gh-user:`SebaJeku` for the tip (:issue:`177`).
 
 .. _IMF:
 

doc/whatsnew.rst

@@ -7,9 +7,14 @@ Next release
 ============
 
 - Adjust the :doc:`example` for current data returned by :ref:`ESTAT <ESTAT>` (:issue:`169`, :pull:`170`).
-- :meth:`.StructureMessage.get` can match on :meth:`.IdentifiableArtefact.urn` (:pull:`170`).
+- Update the base URL of the :ref:`ILO <ILO>` source (:pull:`175`; thanks :gh-user:`SebaJeku` for :issue:`177`).
+- :meth:`.StructureMessage.get` can match on :attr:`.IdentifiableArtefact.urn` (:pull:`170`).
   This makes the method more useful in the case that a message includes artefacts with the same ID but different :attr:`~.MaintainableArtefact.maintainer` and/or :attr:`~.VersionableArtefact.version`.
+- :func:`.urn.make` can handle :class:`.DataConsumerScheme`, :class:`.OrganisationScheme`, :class:`.ReportingTaxonomy`, :class:`.TransformationScheme`, and :class:`.VTLMappingScheme` (:pull:`175`).
+- New method :meth:`.StructureMessage.iter_objects` (:pull:`175`).
+- New method :meth:`.DataMessage.update` (:pull:`175`).
 - Bugfix: :class:`.ItemScheme` could not be :func:`copy.deepcopy` 'd (:pull:`170`).
+- Bugfix: :class:`.TypeError` was raised on :meth:`.Client.get` from an SDMX-JSON source (:pull:`175`).
 
 v2.14.0 (2024-02-20)
 ====================

sdmx/message.py

@@ -13,7 +13,17 @@
 from datetime import datetime
 from itertools import chain
 from operator import attrgetter
-from typing import TYPE_CHECKING, List, Optional, Text, Type, Union, get_args
+from typing import (
+    TYPE_CHECKING,
+    Generator,
+    List,
+    Optional,
+    Text,
+    Tuple,
+    Type,
+    Union,
+    get_args,
+)
 
 from sdmx import model
 from sdmx.dictlike import DictLike, summarize_dictlike
@@ -166,7 +176,7 @@ def __repr__(self):
         lines.extend(_summarize(self, ["footer", "response"]))
         return "\n  ".join(lines)
 
-    def compare(self, other, strict=True):
+    def compare(self, other, strict=True) -> bool:
         """Return :obj:`True` if `self` is the same as `other`.
 
         Two Messages are the same if their :attr:`header` and :attr:`footer` compare
@@ -179,7 +189,7 @@ def compare(self, other, strict=True):
         """
         return self.header.compare(other.header, strict) and (
             self.footer is other.footer is None
-            or self.footer.compare(other.footer, strict)
+            or self.footer.compare(other.footer, strict)  # type: ignore [union-attr]
         )
 
 
@@ -292,6 +302,8 @@ def get(
             of different classes, or two objects of the same class with different
             :attr:`~.MaintainableArtefact.maintainer` or
             :attr:`~.VersionableArtefact.version`.
+
+        .. todo:: Support passing a URN.
         """
         id_ = (
             obj_or_id.id
@@ -313,11 +325,21 @@ def get(
 
         return candidates[0] if len(candidates) == 1 else None
 
-    def iter_collections(self):
+    def iter_collections(self) -> Generator[Tuple[str, type], None, None]:
         """Iterate over collections."""
         for f in direct_fields(self.__class__):
             yield f.name, get_args(f.type)[1]
 
+    def iter_objects(
+        self, external_reference: bool = True
+    ) -> Generator[common.MaintainableArtefact, None, None]:
+        """Iterate over all objects in the message."""
+        for _, cls in self.iter_collections():
+            for obj in self.objects(cls).values():
+                if not external_reference and obj.is_external_reference:
+                    continue
+                yield obj
+
     def objects(self, cls):
         """Get a reference to the attribute for objects of type `cls`.
 
@@ -352,10 +374,9 @@ def __repr__(self):
 class DataMessage(Message):
     """SDMX Data Message.
 
-    .. note:: A DataMessage may contain zero or more :class:`.DataSet`, so
-       :attr:`data` is a list. To retrieve the first (and possibly only)
-       data set in the message, access the first element of the list:
-       ``msg.data[0]``.
+    .. note:: A DataMessage may contain zero or more :class:`.DataSet`, so :attr:`data`
+       is a list. To retrieve the first (and possibly only) data set in the message,
+       access the first element of the list: :py:`msg.data[0]`.
     """
 
     #: :class:`list` of :class:`.DataSet`.
@@ -427,6 +448,74 @@ def compare(self, other, strict=True):
             and all(ds[0].compare(ds[1], strict) for ds in zip(self.data, other.data))
         )
 
+    def update(self) -> None:
+        """Update :attr:`.observation_dimension`.
+
+        The observation dimensions (or dimension observation) is determined
+        automatically if:
+
+        1. There is at least 1 :class:`DataSet <.BaseDataSet>` in the message.
+        2. For at least 1 data set:
+
+           - :attr:`~.BaseDataSet.structured_by` is defined.
+           - There is at least 1 :class:`.Observation` in the data set. (:meth:`.update`
+             checks only the first observation.)
+           - The :attr:`.Observation.dimension` is a :class:`.Key` referring to exactly
+             1 dimension.
+
+        3. The dimension indicated by (2) is the same for all DataSets in the message.
+
+        If not all these conditions are met, messages are logged with level DEBUG, and
+        :attr:`.observation_dimension` is set to :any:`None`.
+
+        .. note:: :meth:`.update` is not automatically called when data sets are added
+           to or removed from :attr:`.data`. User code **should** call :meth:`.update`
+           to reflect such changes.
+        """
+        if not self.data:
+            log.debug("No DataSet in message")
+            self.observation_dimension = None
+            return
+
+        dims = set()
+        for ds in self.data:
+            try:
+                assert ds.structured_by
+
+                # Use the first observation
+                assert len(ds.obs)
+                o0 = ds.obs[0]
+                assert o0.dimension
+
+                # Identify the dimensions specified per-observation
+                d_a_o = tuple(o0.dimension.values.keys())
+
+                if 1 == len(d_a_o):
+                    # Single dimension-at-observation
+                    # Record as an attribute of the DataMessage
+                    dims.add(ds.structured_by.dimensions.get(d_a_o[0]))
+                else:
+                    dims.add(d_a_o)
+            except AssertionError:
+                continue
+
+        if len(dims) == 1 and not all(isinstance(d, tuple) for d in dims):
+            self.observation_dimension = dims.pop()
+        else:
+            if len(dims) == 1:
+                log.debug(f"More than 1 dimension at observation level: {dims.pop()}")
+            elif len(dims) > 1:
+                log.debug(
+                    f"Multiple data sets with different observation dimension: {dims}"
+                )
+            elif not dims:
+                log.debug(
+                    f"Unable to determine observation dimension for {len(self.data)} "
+                    "data set(s). Data set(s) may lack structure reference or "
+                    "observations."
+                )
+            self.observation_dimension = None
+
 
 @dataclass
 class MetadataMessage(DataMessage):

sdmx/model/common.py

@@ -956,9 +956,12 @@ def getdefault(self, id, cls=None, **kwargs) -> CT:
         return component
 
     # Properties of components
-    def __getitem__(self, key) -> CT:
-        """Convenience access to components."""
-        return self.components[key]
+    def __getitem__(self, index: int) -> CT:
+        """Convenience access to :attr:`components` by index.
+
+        To retrieve components by ID, use :meth:`get`.
+        """
+        return self.components[index]
 
     def __len__(self):
         return len(self.components)
@@ -2543,8 +2546,20 @@ class BaseContentConstraint:
 PACKAGE = dict()
 
 _PACKAGE_CLASS: Dict[str, set] = {
-    "base": {"Agency", "AgencyScheme", "DataProvider", "DataProviderScheme"},
-    "categoryscheme": {"Category", "Categorisation", "CategoryScheme"},
+    "base": {
+        "Agency",
+        "AgencyScheme",
+        "DataProvider",
+        "DataConsumerScheme",
+        "DataProviderScheme",
+        "OrganisationScheme",
+    },
+    "categoryscheme": {
+        "Category",
+        "Categorisation",
+        "CategoryScheme",
+        "ReportingTaxonomy",
+    },
     "codelist": {
         "Code",
         "Codelist",
@@ -2571,7 +2586,9 @@ class BaseContentConstraint:
         "CustomTypeScheme",
         "NamePersonalisationScheme",
         "RulesetScheme",
+        "TransformationScheme",
         "UserDefinedOperatorScheme",
+        "VTLMappingScheme",
     },
 }
 

sdmx/reader/base.py

@@ -1,19 +1,23 @@
 import logging
 from abc import ABC, abstractmethod
 from functools import lru_cache
-from typing import List
+from typing import TYPE_CHECKING, ClassVar, List, Optional
+from warnings import warn
 
 from sdmx.format import MediaType
 
+if TYPE_CHECKING:
+    import sdmx.model.common
+
 log = logging.getLogger(__name__)
 
 
 class BaseReader(ABC):
     #: List of media types handled by the reader.
-    media_types: List[MediaType] = []
+    media_types: ClassVar[List[MediaType]] = []
 
     #: List of file name suffixes handled by the reader.
-    suffixes: List[str] = []
+    suffixes: ClassVar[List[str]] = []
 
     @classmethod
     def detect(cls, content: bytes) -> bool:
@@ -41,19 +45,44 @@ def supports_suffix(cls, value: str) -> bool:
         return value.lower() in cls.suffixes
 
     @abstractmethod
-    def read_message(self, source, dsd=None):
+    def read_message(
+        self,
+        source,
+        structure: Optional["sdmx.model.common.Structure"] = None,
+        **kwargs,
+    ):
         """Read message from *source*.
 
         Parameters
         ----------
         source : file-like
             Message content.
-        dsd : :class:`DataStructureDefinition <.BaseDataStructureDefinition>`, optional
-            DSD for aid in reading `source`.
+        structure :
+            :class:`DataStructure <.BaseDataStructureDefinition>` or
+            :class:`MetadataStructure <.BaseMetadataStructureDefinition>`
+            for aid in reading `source`.
 
         Returns
         -------
         :class:`.Message`
             An instance of a Message subclass.
         """
         pass  # pragma: no cover
+
+    @classmethod
+    def _handle_deprecated_kwarg(
+        cls, structure: Optional["sdmx.model.common.Structure"], kwargs
+    ) -> Optional["sdmx.model.common.Structure"]:
+        try:
+            dsd = kwargs.pop("dsd")
+        except KeyError:
+            dsd = None
+        else:
+            warn(
+                "Reader.read_message(…, dsd=…) keyword argument; use structure=…",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            if structure and structure is not dsd:
+                raise ValueError(f"Mismatched structure={structure}, dsd={dsd}")
+        return structure or dsd

sdmx/reader/json.py

@@ -37,10 +37,11 @@ class Reader(BaseReader):
     def detect(cls, content):
         return content.startswith(b"{")
 
-    def read_message(self, source, dsd=None):  # noqa: C901  TODO reduce complexity 15 → ≤11
+    def read_message(self, source, structure=None, **kwargs):  # noqa: C901  TODO reduce complexity 15 → ≤11
         # Initialize message instance
         msg = DataMessage()
 
+        dsd = self._handle_deprecated_kwarg(structure, kwargs)
         if dsd:  # pragma: no cover
             # Store explicit DSD, if any
             msg.dataflow.structure = dsd