Fix yaml seralization for interval

dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION="3.9"
+ARG PYTHON_VERSION="3.10"
 ARG ROOT_CONTAINER=python:${PYTHON_VERSION}-slim-bullseye
 
 
@@ -14,11 +14,11 @@ RUN \
     apt-get -qq upgrade --yes && \
     apt-get -qq install --yes --no-install-recommends \
         build-essential \
-	freetds-dev \
-	git \
-	libssl-dev \
-	libyaml-dev \
-	tini \
+        freetds-dev \
+        git \
+        libssl-dev \
+        libyaml-dev \
+        tini \
     > /dev/null && \
     apt-get -qq clean && \
     rm -rf /var/lib/apt/lists/* && \

pyproject.toml

@@ -31,7 +31,7 @@ classifiers = [
 ]
 dependencies = [
     "blosc>=1.10.6",
-    "cfgenvy>=1.4.1",
+    "cfgenvy>=1.4.2",
     "numpy>=1.15.4",
     "pandas>=0.23.4",
     "requests>=2.26.0",

readme.md

@@ -16,30 +16,37 @@ An opinionated library to help deploy data science projects
 
 ## Develop, Lint & Test
 
-Setup:
+Setup virtual environment:
 
     python3.10 -m venv .venv
 
+Or setup homebrew virtual environment:
+
+    brew install python@3.10
+    /opt/homebrew/Cellar/python@3.10/.../Frameworks/Python.framework/Versions/Current/bin/python@3.10 -m venv .venv
+
+Once virtual environment is setup:
+
     . .venv/bin/activate
-    pip install ".[dev]"
+    pip install -U pip setuptools wheel
+    pip install -e ".[dev]"
     pre-commit install
 
 Session:
 
-    . .env/bin/activate
-    docker-compose up --build postgres &
+    . .venv/bin/activate
+    pytest
     ...
-    CONFIG=./local/test.yaml ENV=./secrets/example.env pre-commit run --all-files
+    pre-commit run --all-files
     ...
-    CONFIG=./local/test.yaml ENV=./secrets/example.env git commit -m 'Message'
+    git commit -m 'Message'
     ...
-    docker-compose down
     deactivate
 
-Rebuild the postgres container and remove the docker volume if the database schema is changed.
-
 ## CI/CD Lint & Test:
 
-    docker-compose up --build test &
+    docker compose up test
+    docker compose up pre-commit
+    docker compose up build-wheel && docker compose up install-wheel
     ...
-    docker-compose down
+    docker compose down

src/dsdk/__init__.py

@@ -3,15 +3,14 @@
 from .asset import Asset
 from .flowsheet import Flowsheet
 from .flowsheet import Mixin as FlowsheetMixin
-from .interval import Interval
+from .interval import Interval, profile
 from .model import Mixin as ModelMixin
 from .model import Model
 from .mssql import Mixin as MssqlMixin
 from .mssql import Persistor as Mssql
 from .postgres import Mixin as PostgresMixin
 from .postgres import Persistor as Postgres
 from .postgres import PredictionMixin as PostgresPredictionMixin
-from .profile import Profile, profile
 from .service import Batch, Delegate, Service, Task
 from .utils import (
     chunks,
@@ -38,7 +37,6 @@
     "PostgresPredictionMixin",
     "PostgresMixin",
     "Postgres",
-    "Profile",
     "Service",
     "Task",
     "chunks",

src/dsdk/flowsheet.py

@@ -16,8 +16,8 @@
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import HTTPError, Timeout
 
+from .interval import Interval, profile
 from .persistor import Persistor
-from .profile import Profile, profile
 from .service import Service
 from .utils import configure_logger, retry
 
@@ -36,7 +36,7 @@ class Result:  # pylint: disable=too-few-public-methods
     def __init__(
         self,
         *,
-        duration: Profile,
+        duration: Interval,
         status: bool,
         description: str | None = None,
         name: str | None = None,
@@ -334,8 +334,8 @@ class Mixin(BaseMixin):
     @classmethod
     def yaml_types(cls) -> None:
         """Yaml types."""
-        super().yaml_types()
         Flowsheet.as_yaml_type()
+        super().yaml_types()
 
     @classmethod
     def publish_flowsheets(cls):

src/dsdk/interval.py

@@ -2,41 +2,63 @@
 
 from __future__ import annotations
 
-from typing import Any
+from contextlib import contextmanager
+from logging import getLogger
+from time import perf_counter_ns
+from typing import Any, Generator
 
-from cfgenvy import yaml_type
+from cfgenvy import YamlMapping
 
+logger = getLogger(__name__)
 
-class Interval:
+
+class Interval(YamlMapping):
     """Interval."""
 
     YAML = "!interval"
 
     @classmethod
-    def as_yaml_type(cls, tag: str | None = None):
-        """As yaml type."""
-        yaml_type(
-            cls,
-            tag or cls.YAML,
-            init=cls._yaml_init,
-            repr=cls._yaml_repr,
-        )
-
-    @classmethod
-    def _yaml_init(cls, loader, node):
-        """Yaml init."""
-        return cls(**loader.construct_mapping(node, deep=True))
+    def _yaml_init(cls, loader, node) -> Any:
+        """Yaml init.
 
-    @classmethod
-    def _yaml_repr(cls, dumper, self, *, tag: str):
-        """Yaml repr."""
-        return dumper.represent_mapping(tag, self.as_yaml())
+        Yaml 1.1 impicit boolean y|n|on|off|yes|no|true|false
+        """
+        a = loader.construct_mapping(node, deep=True)
+        b: dict[str, Any] = {}
+        for key, value in a.items():
+            if key == "'on'" or (key.__class__ == bool and key is True):
+                b["on"] = value
+                continue
+            b[key] = value
+        return cls(**b)  # pylint: disable=missing-kwoa
 
     def __init__(self, *, on: Any, end: Any = None):
         """__init__."""
-        self.on = on
         self.end = end
+        self.on = on
+
+    def __repr__(self):
+        """__repr__."""
+        return f"Interval(end={self.end}, on={self.on})"
 
     def as_yaml(self) -> dict[str, Any]:
         """As yaml."""
         return {"end": self.end, "on": self.on}
+
+
+@contextmanager
+def profile(key: str) -> Generator[Interval, None, None]:
+    """Profile."""
+    # Replace return type with ContextManager[Interval] when mypy is fixed.
+    i = Interval(on=perf_counter_ns())
+    logger.info('{"key": "%s.on", "ns": "%s"}', key, i.on)
+    try:
+        yield i
+    finally:
+        i.end = perf_counter_ns()
+        logger.info(
+            '{"key": "%s.end", "ns": "%s", "elapsed": "%s"}',
+            key,
+            i.end,
+            i.end - i.on,
+        )

src/dsdk/model.py

@@ -38,6 +38,7 @@ class Model:
     @classmethod
     def as_yaml_type(cls, tag: str | None = None) -> None:
         """As yaml type."""
+        logger.debug("%s.as_yaml_type(tag=%s)", cls.__name__, tag)
         yaml_type(
             cls,
             tag or cls.YAML,
@@ -97,12 +98,6 @@ def as_insert_sql(self) -> dict[str, Any]:
 class Mixin(BaseMixin):
     """Mixin."""
 
-    @classmethod
-    def yaml_types(cls) -> None:
-        """As yaml types."""
-        Model.as_yaml_type()
-        super().yaml_types()
-
     def __init__(self, *, model: Model, **kwargs):
         """__init__."""
         self.model = model

src/dsdk/mssql.py

@@ -107,6 +107,7 @@ class Mixin(BaseMixin):
     @classmethod
     def yaml_types(cls) -> None:
         """Yaml types."""
+        logger.debug("dsdk.mssql.Mixin.yaml_types()")
         Persistor.as_yaml_type()
         super().yaml_types()
 

src/dsdk/persistor.py

@@ -16,7 +16,7 @@
 from typing import Any, Generator, Sequence
 
 from blosc import compress, decompress
-from cfgenvy import yaml_type
+from cfgenvy import YamlMapping, yaml_type
 from pandas import DataFrame
 
 from .asset import Asset
@@ -146,11 +146,13 @@ def render(
         resolve = {
             DataFrame: cls.union_all_df,
         }
-        keys = {
-            key: resolve.get(sequence.__class__, cls.union_all)(cur, sequence)
-            for key, sequence in keys.items()
+        union_alls = {
+            key: resolve.get(value.__class__, cls.union_all)(cur, value)
+            for key, value in keys.items()
         }
-        query = query.format(**keys)
+        logger.info("Union_alls: %s", union_alls)
+        logger.info("Query: %s", query)
+        query = query.format(**union_alls)
         return cls.mogrify(cur, query, parameters).decode("utf-8")
 
     @classmethod
@@ -281,10 +283,10 @@ def rollback(self) -> Generator[Any, None, None]:
                 logger.info(self.ROLLBACK)
 
 
-class Persistor(AbstractPersistor):
+class Persistor(YamlMapping, AbstractPersistor):
     """Persistor."""
 
-    YAML = "!basepersistor"
+    YAML = "!persistor"
 
     @classmethod
     def as_yaml_type(cls, tag: str | None = None) -> None:
@@ -297,16 +299,6 @@ def as_yaml_type(cls, tag: str | None = None) -> None:
             repr=cls._yaml_repr,
         )
 
-    @classmethod
-    def _yaml_init(cls, loader, node):
-        """Yaml init."""
-        return cls(**loader.construct_mapping(node, deep=True))
-
-    @classmethod
-    def _yaml_repr(cls, dumper, self, *, tag: str):
-        """Yaml repr."""
-        return dumper.represent_mapping(tag, self.as_yaml())
-
     def __init__(  # pylint: disable=too-many-arguments
         self,
         *,

src/dsdk/postgres.py

@@ -335,6 +335,7 @@ class Mixin(BaseMixin):
     @classmethod
     def yaml_types(cls) -> None:
         """Yaml types."""
+        logger.debug("dsdk.postgres.Mixin.yaml_types()")
         Persistor.as_yaml_type()
         super().yaml_types()