🎨 extend ooil to support depends_on keyword in overwrites (#7041)

Co-authored-by: Andrei Neagu <neagu@itis.swiss>

packages/models-library/src/models_library/user_preferences.py

@@ -98,30 +98,20 @@ def to_db(self) -> dict:
 
     @classmethod
     def update_preference_default_value(cls, new_default: Any) -> None:
-        expected_type = get_type(
-            cls.model_fields["value"]  # pylint: disable=unsubscriptable-object
-        )
+        # pylint: disable=unsubscriptable-object
+        expected_type = get_type(cls.model_fields["value"])
         detected_type = type(new_default)
         if expected_type != detected_type:
             msg = (
                 f"Error, {cls.__name__} {expected_type=} differs from {detected_type=}"
             )
             raise TypeError(msg)
 
-        if (
-            cls.model_fields["value"].default  # pylint: disable=unsubscriptable-object
-            is None
-        ):
-            cls.model_fields[  # pylint: disable=unsubscriptable-object
-                "value"
-            ].default_factory = lambda: new_default
+        if cls.model_fields["value"].default is None:
+            cls.model_fields["value"].default_factory = lambda: new_default
         else:
-            cls.model_fields[  # pylint: disable=unsubscriptable-object
-                "value"
-            ].default = new_default
-            cls.model_fields[  # pylint: disable=unsubscriptable-object
-                "value"
-            ].default_factory = None
+            cls.model_fields["value"].default = new_default
+            cls.model_fields["value"].default_factory = None
 
         cls.model_rebuild(force=True)
 

packages/models-library/src/models_library/utils/labels_annotations.py

@@ -10,7 +10,7 @@
 
 from common_library.json_serialization import json_dumps
 
-LabelsAnnotationsDict: TypeAlias = dict[str, str]
+LabelsAnnotationsDict: TypeAlias = dict[str, str | float | bool | None]
 
 # SEE https://docs.docker.com/config/labels-custom-metadata/#label-keys-and-values
 #  "Authors of third-party tools should prefix each label key with the reverse DNS notation of a
@@ -28,7 +28,7 @@ def to_labels(
     """converts config into labels annotations"""
 
     # FIXME: null is loaded as 'null' string value? is that correct? json -> None upon deserialization?
-    labels = {}
+    labels: LabelsAnnotationsDict = {}
     for key, value in config.items():
         if trim_key_head:
             if isinstance(value, str):
@@ -57,7 +57,7 @@ def from_labels(
     for key, label in labels.items():
         if key.startswith(f"{prefix_key}."):
             try:
-                value = json.loads(label)
+                value = json.loads(label)  # type: ignore
             except JSONDecodeError:
                 value = label
 

packages/service-integration/requirements/prod.txt

@@ -9,6 +9,7 @@
 # installs base + tests requirements
 --requirement _base.txt
 
+simcore-common-library @ ../common-library/
 simcore-models-library @ ../models-library
 
 # current module

packages/service-integration/src/service_integration/cli/_compose_spec.py

@@ -6,7 +6,7 @@
 import rich
 import typer
 import yaml
-from models_library.utils.labels_annotations import to_labels
+from models_library.utils.labels_annotations import LabelsAnnotationsDict, to_labels
 from rich.console import Console
 from yarl import URL
 
@@ -93,7 +93,7 @@ def create_docker_compose_image_spec(
             rich.print("No runtime config found (optional), using default.")
 
     # OCI annotations (optional)
-    extra_labels = {}
+    extra_labels: LabelsAnnotationsDict = {}
     try:
         oci_spec = yaml.safe_load(
             (config_basedir / f"{OCI_LABEL_PREFIX}.yml").read_text()

packages/service-integration/src/service_integration/cli/_config.py

@@ -5,6 +5,7 @@
 import rich
 import typer
 import yaml
+from models_library.utils.labels_annotations import LabelsAnnotationsDict
 from pydantic import BaseModel
 
 from ..compose_spec_model import ComposeSpecification
@@ -20,7 +21,7 @@
 )
 
 
-def _get_labels_or_raise(build_labels) -> dict[str, str]:
+def _get_labels_or_raise(build_labels) -> LabelsAnnotationsDict:
     if isinstance(build_labels, list):
         return dict(item.strip().split("=") for item in build_labels)
     if isinstance(build_labels, dict):
@@ -56,7 +57,9 @@ def _save(service_name: str, filename: Path, model: BaseModel):
             rich.print(f"Creating {output_path} ...", end="")
 
             with output_path.open("wt") as fh:
-                data = json.loads(model.model_dump_json(by_alias=True, exclude_none=True))
+                data = json.loads(
+                    model.model_dump_json(by_alias=True, exclude_none=True)
+                )
                 yaml.safe_dump(data, fh, sort_keys=False)
 
             rich.print("DONE")
@@ -68,7 +71,7 @@ def _save(service_name: str, filename: Path, model: BaseModel):
                     service_name
                 ].build.labels:  # AttributeError if build is str
 
-                    labels: dict[str, str] = _get_labels_or_raise(build_labels)
+                    labels = _get_labels_or_raise(build_labels)
                     meta_cfg = MetadataConfig.from_labels_annotations(labels)
                     _save(service_name, metadata_path, meta_cfg)
 
@@ -86,11 +89,7 @@ def _save(service_name: str, filename: Path, model: BaseModel):
                     runtime_cfg = RuntimeConfig.from_labels_annotations(labels)
                     _save(service_name, service_specs_path, runtime_cfg)
 
-            except (  # noqa: PERF203
-                AttributeError,
-                TypeError,
-                ValueError,
-            ) as err:
+            except (AttributeError, TypeError, ValueError) as err:
                 rich.print(
                     f"WARNING: failure producing specs for {service_name}: {err}"
                 )

packages/service-integration/src/service_integration/compose_spec_model.py

@@ -10,6 +10,7 @@
 from ._compose_spec_model_autogenerated import (  # type:ignore
     BuildItem,
     ComposeSpecification,
+    ListOrDict,
     Service,
     Volume1,
 )
@@ -23,6 +24,7 @@
 __all__: tuple[str, ...] = (
     "BuildItem",
     "ComposeSpecification",
+    "ListOrDict",
     "SCHEMA_VERSION",
     "Service",
     "ServiceVolume",

packages/service-integration/src/service_integration/oci_image_spec.py

@@ -10,7 +10,11 @@
 from typing import Annotated, Any
 
 from models_library.basic_types import SHA1Str, VersionStr
-from models_library.utils.labels_annotations import from_labels, to_labels
+from models_library.utils.labels_annotations import (
+    LabelsAnnotationsDict,
+    from_labels,
+    to_labels,
+)
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.networks import AnyUrl
 
@@ -132,17 +136,16 @@ class OciImageSpecAnnotations(BaseModel):
 
     @classmethod
     def from_labels_annotations(
-        cls, labels: dict[str, str]
+        cls, labels: LabelsAnnotationsDict
     ) -> "OciImageSpecAnnotations":
         data = from_labels(labels, prefix_key=OCI_LABEL_PREFIX, trim_key_head=False)
         return cls.model_validate(data)
 
-    def to_labels_annotations(self) -> dict[str, str]:
-        labels: dict[str, str] = to_labels(
+    def to_labels_annotations(self) -> LabelsAnnotationsDict:
+        return to_labels(
             self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True),
             prefix_key=OCI_LABEL_PREFIX,
         )
-        return labels
 
 
 class LabelSchemaAnnotations(BaseModel):
@@ -164,7 +167,7 @@ class LabelSchemaAnnotations(BaseModel):
     @classmethod
     def create_from_env(cls) -> "LabelSchemaAnnotations":
         data = {}
-        for field_name in cls.model_fields.keys():
+        for field_name in cls.model_fields.keys():  # noqa: SIM118
             if value := os.environ.get(field_name.upper()):
                 data[field_name] = value
         return cls.model_validate(data)

packages/service-integration/src/service_integration/osparc_config.py

@@ -34,12 +34,14 @@
 from models_library.services_types import ServiceKey
 from models_library.utils.labels_annotations import (
     OSPARC_LABEL_PREFIXES,
+    LabelsAnnotationsDict,
     from_labels,
     to_labels,
 )
 from pydantic import (
     ConfigDict,
     NonNegativeInt,
+    TypeAdapter,
     ValidationError,
     ValidationInfo,
     field_validator,
@@ -121,24 +123,21 @@ def _check_contact_in_authors(cls, v, info: ValidationInfo):
     def from_yaml(cls, path: Path) -> "MetadataConfig":
         with path.open() as fh:
             data = yaml_safe_load(fh)
-        model: "MetadataConfig" = cls.model_validate(data)
-        return model
+        return cls.model_validate(data)
 
     @classmethod
-    def from_labels_annotations(cls, labels: dict[str, str]) -> "MetadataConfig":
+    def from_labels_annotations(cls, labels: LabelsAnnotationsDict) -> "MetadataConfig":
         data = from_labels(
             labels, prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False
         )
-        model: "MetadataConfig" = cls.model_validate(data)
-        return model
+        return cls.model_validate(data)
 
-    def to_labels_annotations(self) -> dict[str, str]:
-        labels: dict[str, str] = to_labels(
+    def to_labels_annotations(self) -> LabelsAnnotationsDict:
+        return to_labels(
             self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True),
             prefix_key=OSPARC_LABEL_PREFIXES[0],
             trim_key_head=False,
         )
-        return labels
 
     def service_name(self) -> str:
         """name used as key in the compose-spec services map"""
@@ -151,7 +150,9 @@ def image_name(self, settings: AppSettings, registry="local") -> str:
         if registry in "dockerhub":
             # dockerhub allows only one-level names -> dot it
             # TODO: check thisname is compatible with REGEX
-            service_path = ServiceKey(service_path.replace("/", "."))
+            service_path = TypeAdapter(ServiceKey).validate_python(
+                service_path.replace("/", ".")
+            )
 
         service_version = self.version
         return f"{registry_prefix}{service_path}:{service_version}"
@@ -264,13 +265,12 @@ def from_yaml(cls, path: Path) -> "RuntimeConfig":
         return cls.model_validate(data)
 
     @classmethod
-    def from_labels_annotations(cls, labels: dict[str, str]) -> "RuntimeConfig":
+    def from_labels_annotations(cls, labels: LabelsAnnotationsDict) -> "RuntimeConfig":
         data = from_labels(labels, prefix_key=OSPARC_LABEL_PREFIXES[1])
         return cls.model_validate(data)
 
-    def to_labels_annotations(self) -> dict[str, str]:
-        labels: dict[str, str] = to_labels(
+    def to_labels_annotations(self) -> LabelsAnnotationsDict:
+        return to_labels(
             self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True),
             prefix_key=OSPARC_LABEL_PREFIXES[1],
         )
-        return labels

packages/service-integration/src/service_integration/osparc_image_specs.py

@@ -2,10 +2,13 @@
 
 """
 
+from typing import Any
 
+from models_library.utils.labels_annotations import LabelsAnnotationsDict
 from service_integration.compose_spec_model import (
     BuildItem,
     ComposeSpecification,
+    ListOrDict,
     Service,
 )
 
@@ -19,14 +22,14 @@ def create_image_spec(
     docker_compose_overwrite_cfg: DockerComposeOverwriteConfig,
     runtime_cfg: RuntimeConfig | None = None,
     *,
-    extra_labels: dict[str, str] | None = None,
+    extra_labels: LabelsAnnotationsDict | None = None,
     **_context
 ) -> ComposeSpecification:
     """Creates the image-spec provided the osparc-config and a given context (e.g. development)
 
     - the image-spec simplifies building an image to ``docker compose build``
     """
-    labels = {**meta_cfg.to_labels_annotations()}
+    labels = meta_cfg.to_labels_annotations()
     if extra_labels:
         labels.update(extra_labels)
     if runtime_cfg:
@@ -36,19 +39,26 @@ def create_image_spec(
 
     assert docker_compose_overwrite_cfg.services  # nosec
 
-    if not docker_compose_overwrite_cfg.services[service_name].build.context:
-        docker_compose_overwrite_cfg.services[service_name].build.context = "./"
+    build = docker_compose_overwrite_cfg.services[service_name].build
+    assert isinstance(build, BuildItem)  # nosec
+    if not build.context:
+        build.context = "./"
 
-    docker_compose_overwrite_cfg.services[service_name].build.labels = labels
+    build.labels = ListOrDict(root=labels)
 
-    overwrite_options = docker_compose_overwrite_cfg.services[
-        service_name
-    ].build.model_dump(exclude_none=True, serialize_as_any=True)
+    overwrite_options = build.model_dump(exclude_none=True, serialize_as_any=True)
     build_spec = BuildItem(**overwrite_options)
 
+    service_kwargs: dict[str, Any] = {
+        "image": meta_cfg.image_name(settings),
+        "build": build_spec,
+    }
+    if docker_compose_overwrite_cfg.services[service_name].depends_on:
+        service_kwargs["depends_on"] = docker_compose_overwrite_cfg.services[
+            service_name
+        ].depends_on
+
     return ComposeSpecification(
         version=settings.COMPOSE_VERSION,
-        services={
-            service_name: Service(image=meta_cfg.image_name(settings), build=build_spec)
-        },
+        services={service_name: Service(**service_kwargs)},
     )

packages/service-integration/tests/data/docker-compose.overwrite.yml

@@ -1,4 +1,6 @@
 services:
   osparc-python-runner:
+    depends_on:
+      - another-service
     build:
       dockerfile: Dockerfile

packages/service-integration/tests/test_command_compose.py

@@ -3,14 +3,22 @@
 # pylint: disable=unused-variable
 
 import os
+import traceback
 from collections.abc import Callable
 from pathlib import Path
 
 import yaml
+from click.testing import Result
 from service_integration.compose_spec_model import ComposeSpecification
 from service_integration.osparc_config import MetadataConfig
 
 
+def _format_cli_error(result: Result) -> str:
+    assert result.exception
+    tb_message = "\n".join(traceback.format_tb(result.exception.__traceback__))
+    return f"Below exception was raised by the cli:\n{tb_message}"
+
+
 def test_make_docker_compose_meta(
     run_program_with_args: Callable,
     docker_compose_overwrite_path: Path,
@@ -33,7 +41,7 @@ def test_make_docker_compose_meta(
         "--to-spec-file",
         target_compose_specs,
     )
-    assert result.exit_code == os.EX_OK, result.output
+    assert result.exit_code == os.EX_OK, _format_cli_error(result)
 
     # produces a compose spec
     assert target_compose_specs.exists()
@@ -50,6 +58,4 @@ def test_make_docker_compose_meta(
     assert compose_labels
     assert isinstance(compose_labels.root, dict)
 
-    assert (
-        MetadataConfig.from_labels_annotations(compose_labels.root) == metadata_cfg
-    )
+    assert MetadataConfig.from_labels_annotations(compose_labels.root) == metadata_cfg