From 706dcfc1c67e18d116117713bbd2c643ed41f610 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:23:52 +0200 Subject: [PATCH 01/20] directorv2 models -> models_library --- .../api_schemas_directorv2/__init__.py | 0 .../api_schemas_directorv2/clusters.py | 203 ++++++++ .../api_schemas_directorv2/constants.py | 22 + .../dynamic_services.py | 63 +++ .../dynamic_services_scheduler.py | 485 ++++++++++++++++++ .../dynamic_services_service.py | 153 ++++++ .../api_schemas_directorv2/health.py | 14 + .../api_schemas_directorv2/meta.py | 27 + 8 files changed, 967 insertions(+) create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/__init__.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/clusters.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/constants.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/health.py create mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/meta.py diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/__init__.py b/packages/models-library/src/models_library/api_schemas_directorv2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py new file mode 100644 index 00000000000..2aa49f03ccb --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -0,0 +1,203 @@ +from typing import Any, ClassVar, TypeAlias + +from pydantic import ( + AnyHttpUrl, + BaseModel, + Field, + HttpUrl, + NonNegativeFloat, + root_validator, + validator, +) +from pydantic.networks import AnyUrl +from pydantic.types import ByteSize, PositiveFloat + +from ..clusters import ( + CLUSTER_ADMIN_RIGHTS, + CLUSTER_MANAGER_RIGHTS, + CLUSTER_USER_RIGHTS, + BaseCluster, + Cluster, + ClusterAccessRights, + ClusterAuthentication, + ClusterTypeInModel, + ExternalClusterAuthentication, +) +from ..generics import DictModel +from ..users import GroupID + + +class TaskCounts(BaseModel): + error: int = 0 + memory: int = 0 + executing: int = 0 + + +class WorkerMetrics(BaseModel): + cpu: float = Field(..., description="consumed % of cpus") + memory: ByteSize = Field(..., description="consumed memory") + num_fds: int = Field(..., description="consumed file descriptors") + task_counts: TaskCounts = Field(..., description="task details") + + +AvailableResources: TypeAlias = DictModel[str, PositiveFloat] + + +class UsedResources(DictModel[str, NonNegativeFloat]): + @root_validator(pre=True) + @classmethod + def ensure_negative_value_is_zero(cls, values): + # dasks adds/remove resource values and sometimes + # they end up being negative instead of 0 + if v := values.get("__root__", {}): + for res_key, res_value in v.items(): + if res_value < 0: + v[res_key] = 0 + return values + + +class Worker(BaseModel): + id: str + name: str + resources: AvailableResources + used_resources: UsedResources + memory_limit: ByteSize + metrics: WorkerMetrics + + +WorkersDict: TypeAlias = dict[AnyUrl, Worker] + + +class Scheduler(BaseModel): + status: str = Field(..., description="The running status of the scheduler") + workers: WorkersDict | None = Field(default_factory=dict) + + @validator("workers", pre=True, always=True) + @classmethod + def ensure_workers_is_empty_dict(cls, v): + if v is None: + return {} + return v + + +class ClusterDetails(BaseModel): + scheduler: Scheduler = Field( + ..., + description="This contains dask scheduler information given by the underlying dask library", + ) + dashboard_link: AnyUrl = Field( + ..., description="Link to this scheduler's dashboard" + ) + + +class ClusterGet(Cluster): + access_rights: dict[GroupID, ClusterAccessRights] = Field( + alias="accessRights", default_factory=dict + ) + + class Config(Cluster.Config): + allow_population_by_field_name = True + + @root_validator(pre=True) + @classmethod + def ensure_access_rights_converted(cls, values): + if "access_rights" in values: + access_rights = values.pop("access_rights") + values["accessRights"] = access_rights + return values + + +class ClusterDetailsGet(ClusterDetails): + ... + + +class ClusterCreate(BaseCluster): + owner: GroupID | None + authentication: ExternalClusterAuthentication + access_rights: dict[GroupID, ClusterAccessRights] = Field( + alias="accessRights", default_factory=dict + ) + + @validator("thumbnail", always=True, pre=True) + @classmethod + def set_default_thumbnail_if_empty(cls, v, values): + if v is None: + cluster_type = values["type"] + default_thumbnails = { + ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", + ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", + } + return default_thumbnails[cluster_type] + return v + + class Config(BaseCluster.Config): + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + "name": "My awesome cluster", + "type": ClusterTypeInModel.ON_PREMISE, + "endpoint": "https://registry.osparc-development.fake.dev", + "authentication": { + "type": "simple", + "username": "someuser", + "password": "somepassword", + }, + }, + { + "name": "My AWS cluster", + "description": "a AWS cluster administered by me", + "type": ClusterTypeInModel.AWS, + "owner": 154, + "endpoint": "https://registry.osparc-development.fake.dev", + "authentication": { + "type": "simple", + "username": "someuser", + "password": "somepassword", + }, + "accessRights": { + 154: CLUSTER_ADMIN_RIGHTS, + 12: CLUSTER_MANAGER_RIGHTS, + 7899: CLUSTER_USER_RIGHTS, + }, + }, + ] + } + + +class ClusterPatch(BaseCluster): + name: str | None + description: str | None + type: ClusterTypeInModel | None + owner: GroupID | None + thumbnail: HttpUrl | None + endpoint: AnyUrl | None + authentication: ExternalClusterAuthentication | None + access_rights: dict[GroupID, ClusterAccessRights] | None = Field( + alias="accessRights" + ) + + class Config(BaseCluster.Config): + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + "name": "Changing the name of my cluster", + }, + { + "description": "adding a better description", + }, + { + "accessRights": { + 154: CLUSTER_ADMIN_RIGHTS, + 12: CLUSTER_MANAGER_RIGHTS, + 7899: CLUSTER_USER_RIGHTS, + }, + }, + ] + } + + +class ClusterPing(BaseModel): + endpoint: AnyHttpUrl + authentication: ClusterAuthentication = Field( + ..., description="Dask gateway authentication" + ) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/constants.py b/packages/models-library/src/models_library/api_schemas_directorv2/constants.py new file mode 100644 index 00000000000..8a2b7e8e94c --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/constants.py @@ -0,0 +1,22 @@ +from typing import Final + +# dynamic services + +DYNAMIC_SIDECAR_SERVICE_PREFIX: Final[str] = "dy-sidecar" +DYNAMIC_PROXY_SERVICE_PREFIX: Final[str] = "dy-proxy" +DYNAMIC_VOLUME_REMOVER_PREFIX: Final[str] = "dy-volrm" + +# label storing scheduler_data to allow service +# monitoring recovery after director-v2 reboots +DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: Final[str] = "io.simcore.scheduler-data" + +# This matches registries by: +# - local +# - itisfoundation +# - 10.0.0.0:8473 (IP & Port) +DYNAMIC_SIDECAR_DOCKER_IMAGE_RE = ( + r"(^([_a-zA-Z0-9:.-]+)/)?(dynamic-sidecar):([_a-zA-Z0-9.-]+$)" +) + +REGEX_DY_SERVICE_SIDECAR = rf"^{DYNAMIC_SIDECAR_SERVICE_PREFIX}_[a-zA-Z0-9-_]*" +REGEX_DY_SERVICE_PROXY = rf"^{DYNAMIC_PROXY_SERVICE_PREFIX}_[a-zA-Z0-9-_]*" diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py new file mode 100644 index 00000000000..58fe0fdd6e8 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py @@ -0,0 +1,63 @@ +from typing import Any, ClassVar, TypeAlias + +from pydantic import BaseModel, ByteSize, Field + +from ..services import ServicePortKey +from ..services_resources import ServiceResourcesDict, ServiceResourcesDictHelpers +from .dynamic_services_service import RunningDynamicServiceDetails, ServiceDetails + + +class RetrieveDataIn(BaseModel): + port_keys: list[ServicePortKey] = Field( + ..., description="The port keys to retrieve data from" + ) + + +class RetrieveDataOut(BaseModel): + size_bytes: ByteSize = Field( + ..., description="The amount of data transferred by the retrieve call" + ) + + +class RetrieveDataOutEnveloped(BaseModel): + data: RetrieveDataOut + + @classmethod + def from_transferred_bytes( + cls, transferred_bytes: int + ) -> "RetrieveDataOutEnveloped": + return cls(data=RetrieveDataOut(size_bytes=ByteSize(transferred_bytes))) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [{"data": {"size_bytes": 42}}] + } + + +class DynamicServiceCreate(ServiceDetails): + service_resources: ServiceResourcesDict + + product_name: str = Field(..., description="Current product name") + can_save: bool = Field( + ..., description="the service data must be saved when closing" + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "key": "simcore/services/dynamic/3dviewer", + "version": "2.4.5", + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "product_name": "osparc", + "can_save": True, + "service_resources": ServiceResourcesDictHelpers.Config.schema_extra[ + "examples" + ][0], + } + } + + +DynamicServiceGet: TypeAlias = RunningDynamicServiceDetails diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py new file mode 100644 index 00000000000..9778a3b7c4f --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py @@ -0,0 +1,485 @@ +import json +import logging +import re +from collections.abc import Mapping +from enum import Enum +from functools import cached_property +from typing import Any, TypeAlias +from uuid import UUID + +from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as +from servicelib.error_codes import ErrorCodeStr +from servicelib.exception_utils import DelayedExceptionHandler + +from ..api_schemas_directorv2.dynamic_services_service import CommonServiceDetails +from ..basic_types import PortInt +from ..generated_models.docker_rest_api import ContainerState, Status2 +from ..projects_nodes_io import NodeID +from ..service_settings_labels import ( + DynamicSidecarServiceLabels, + PathMappingsLabel, + SimcoreServiceLabels, +) +from ..services import RunID +from ..services_resources import ServiceResourcesDict +from .constants import ( + DYNAMIC_PROXY_SERVICE_PREFIX, + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, + REGEX_DY_SERVICE_PROXY, + REGEX_DY_SERVICE_SIDECAR, +) + +TEMPORARY_PORT_NUMBER = 65_534 + +MAX_ALLOWED_SERVICE_NAME_LENGTH: int = 63 + + +DockerStatus: TypeAlias = Status2 + + +class DockerId(ConstrainedStr): + max_length = 25 + regex = re.compile(r"[A-Za-z0-9]{25}") + + +ServiceId: TypeAlias = DockerId +NetworkId: TypeAlias = DockerId + + +class ServiceName(ConstrainedStr): + strip_whitespace = True + min_length = 2 + + +logger = logging.getLogger() + + +def _strip_service_name(service_name: str) -> str: + """returns: the maximum allowed service name in docker swarm""" + return service_name[:MAX_ALLOWED_SERVICE_NAME_LENGTH] + + +def assemble_service_name(service_prefix: str, node_uuid: NodeID) -> str: + return _strip_service_name("_".join([service_prefix, str(node_uuid)])) + + +class DynamicSidecarStatus(str, Enum): + OK = "ok" # running as expected + FAILING = "failing" # requests to the sidecar API are failing service should be cosnidered as unavailable + + +class Status(BaseModel): + """Generated from data from docker container inspect API""" + + current: DynamicSidecarStatus = Field(..., description="status of the service") + info: str = Field(..., description="additional information for the user") + + def _update(self, new_status: DynamicSidecarStatus, new_info: str) -> None: + self.current = new_status + self.info = new_info + + def update_ok_status(self, info: str) -> None: + self._update(DynamicSidecarStatus.OK, info) + + def update_failing_status( + self, user_msg: str, error_code: ErrorCodeStr | None = None + ) -> None: + next_info = f"{user_msg}" + if error_code: + next_info = f"{user_msg} [{error_code}]" + + self._update(DynamicSidecarStatus.FAILING, next_info) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Status): + return NotImplemented + return self.current == other.current and self.info == other.info + + @classmethod + def create_as_initially_ok(cls) -> "Status": + # the service is initially ok when started + initial_state: "Status" = cls(current=DynamicSidecarStatus.OK, info="") + return initial_state + + +class DockerContainerInspect(BaseModel): + container_state: ContainerState = Field( + ..., description="current state of container" + ) + name: str = Field(..., description="docker name of the container") + id: str = Field(..., description="docker id of the container") + + @cached_property + def status(self) -> DockerStatus: + assert self.container_state.Status # nosec + result: DockerStatus = self.container_state.Status + return result + + @classmethod + def from_container(cls, container: dict[str, Any]) -> "DockerContainerInspect": + return cls( + container_state=ContainerState(**container["State"]), + name=container["Name"], + id=container["Id"], + ) + + class Config: + keep_untouched = (cached_property,) + allow_mutation = False + + +class ServiceRemovalState(BaseModel): + can_remove: bool = Field( + default=False, + description="when True, marks the service as ready to be removed", + ) + can_save: bool = Field( + default=False, + description="when True, saves the internal state and upload outputs of the service", + ) + was_removed: bool = Field( + default=False, + description=( + "Will be True when the removal finished. Used primarily " + "to cancel retrying long running operations." + ), + ) + + def mark_to_remove(self, *, can_save: bool) -> None: + self.can_remove = True + self.can_save = can_save + + def mark_removed(self) -> None: + self.can_remove = False + self.was_removed = True + + +class DynamicSidecar(BaseModel): + status: Status = Field( + Status.create_as_initially_ok(), + description="status of the service sidecar also with additional information", + ) + + is_ready: bool = Field( + default=False, + scription=( + "is True while the health check on the dynamic-sidecar is responding. " + "Meaning that the dynamic-sidecar is reachable and can accept requests" + ), + ) + + @property + def compose_spec_submitted(self) -> bool: + """ + If the director-v2 is rebooted was_compose_spec_submitted is False + If the compose-spec is submitted it can be safely assumed that the + containers_inspect contains some elements. + """ + return self.was_compose_spec_submitted or len(self.containers_inspect) > 0 + + was_compose_spec_submitted: bool = Field( + default=False, + description="if the docker-compose spec was already submitted this fields is True", + ) + + containers_inspect: list[DockerContainerInspect] = Field( + [], + scription="docker inspect results from all the container ran at regular intervals", + ) + + was_dynamic_sidecar_started: bool = False + is_healthy: bool = False + were_containers_created: bool = Field( + default=False, + description=( + "when True no longer will the Docker api " + "be used to check if the services were started" + ), + ) + is_project_network_attached: bool = Field( + default=False, + description=( + "When True, all containers were in running state and project " + "networks were attached. Waiting for the container sto be in " + "running state guarantees all containers have been created" + ), + ) + + is_service_environment_ready: bool = Field( + default=False, + description=( + "True when the environment setup required by the " + "dynamic-sidecars created services was completed." + "Example: nodeports data downloaded, globally " + "shared service data fetched, etc.." + ), + ) + + service_removal_state: ServiceRemovalState = Field( + default_factory=ServiceRemovalState, + description=( + "stores information used during service removal " + "from the dynamic-sidecar scheduler" + ), + ) + + wait_for_manual_intervention_after_error: bool = Field( + default=False, + description=( + "Marks the sidecar as untouchable since there was an error and " + "important data might be lost. awaits for manual intervention." + ), + ) + wait_for_manual_intervention_logged: bool = Field( + default=False, description="True if a relative message was logged" + ) + were_state_and_outputs_saved: bool = Field( + default=False, + description="set True if the dy-sidecar saves the state and uploads the outputs", + ) + + # below had already been validated and + # used only to start the proxy + dynamic_sidecar_id: ServiceId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + dynamic_sidecar_network_id: NetworkId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + swarm_network_id: NetworkId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + swarm_network_name: str | None = Field( + default=None, description="used for starting the proxy" + ) + + docker_node_id: str | None = Field( + default=None, + description=( + "contains node id of the docker node where all services " + "and created containers are started" + ), + ) + + inspect_error_handler: DelayedExceptionHandler = Field( + default=DelayedExceptionHandler(delay_for=0), + description=( + "Set when the dy-sidecar can no longer be reached by the " + "director-v2. If it will be possible to reach the dy-sidecar again, " + "this value will be set to None." + ), + ) + + class Config: + validate_assignment = True + + +class DynamicSidecarNamesHelper(BaseModel): + """ + Service naming schema: + NOTE: name is max 63 characters + dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf + + dynamic sidecar structure + 0. a network is created: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + 1. a dynamic-sidecar is started: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + a traefik instance: dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf + """ + + service_name_dynamic_sidecar: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, + description="unique name of the dynamic-sidecar service", + ) + proxy_service_name: str = Field( + ..., + regex=REGEX_DY_SERVICE_PROXY, + max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, + description="name of the proxy for the dynamic-sidecar", + ) + + simcore_traefik_zone: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + description="unique name for the traefik constraints", + ) + dynamic_sidecar_network_name: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + description="based on the node_id and project_id", + ) + + @classmethod + def make(cls, node_uuid: UUID) -> "DynamicSidecarNamesHelper": + return cls( + service_name_dynamic_sidecar=assemble_service_name( + DYNAMIC_SIDECAR_SERVICE_PREFIX, node_uuid + ), + proxy_service_name=assemble_service_name( + DYNAMIC_PROXY_SERVICE_PREFIX, node_uuid + ), + simcore_traefik_zone=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", + dynamic_sidecar_network_name=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", + ) + + +class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): + # TODO: ANE this object is just the context of the dy-sidecar. Should + # be called like so and subcontexts for different handlers should + # also be added. It will make keeping track of env vars more easily + + service_name: ServiceName = Field( + ..., + description="Name of the current dynamic-sidecar being observed", + ) + run_id: RunID = Field( + default_factory=RunID.create, + description=( + "Uniquely identify the dynamic sidecar session (a.k.a. 2 " + "subsequent exact same services will have a different run_id)" + ), + ) + hostname: str = Field( + ..., description="dy-sidecar's service hostname (provided by docker-swarm)" + ) + port: PortInt = Field( + default=parse_obj_as(PortInt, 8000), description="dynamic-sidecar port" + ) + + @property + def endpoint(self) -> AnyHttpUrl: + """endpoint where all the services are exposed""" + url: AnyHttpUrl = parse_obj_as( + AnyHttpUrl, f"http://{self.hostname}:{self.port}" # NOSONAR + ) + return url + + dynamic_sidecar: DynamicSidecar = Field( + ..., + description="stores information fetched from the dynamic-sidecar", + ) + + paths_mapping: PathMappingsLabel # overwrites in DynamicSidecarServiceLabels + + dynamic_sidecar_network_name: str = Field( + ..., + description="overlay network biding the proxy to the container spaned by the dynamic-sidecar", + ) + + simcore_traefik_zone: str = Field( + ..., + description="required for Traefik to correctly route requests to the spawned container", + ) + + service_port: PortInt = Field( + default=parse_obj_as(PortInt, TEMPORARY_PORT_NUMBER), + description=( + "port where the service is exposed defined by the service; " + "NOTE: temporary default because it will be changed once the service " + "is started, this value is fetched from the service start spec" + ), + ) + + service_resources: ServiceResourcesDict = Field( + ..., description="service resources used to enforce limits" + ) + + request_dns: str = Field( + ..., description="used when configuring the CORS options on the proxy" + ) + request_scheme: str = Field( + ..., description="used when configuring the CORS options on the proxy" + ) + request_simcore_user_agent: str = Field( + ..., + description="used as label to filter out the metrics from the cAdvisor prometheus metrics", + ) + proxy_service_name: str = Field( + default=None, description="service name given to the proxy" + ) + proxy_admin_api_port: PortInt | None = Field( + default=None, description="used as the admin endpoint API port" + ) + + @property + def get_proxy_endpoint(self) -> AnyHttpUrl: + """get the endpoint where the proxy's admin API is exposed""" + assert self.proxy_admin_api_port # nosec + url: AnyHttpUrl = parse_obj_as( + AnyHttpUrl, + f"http://{self.proxy_service_name}:{self.proxy_admin_api_port}", # nosec # NOSONAR + ) + return url + + product_name: str = Field( + None, + description="Current product upon which this service is scheduled. " + "If set to None, the current product is undefined. Mostly for backwards compatibility", + ) + + @classmethod + def from_http_request( + # pylint: disable=too-many-arguments + cls, + service: "DynamicServiceCreate", # type: ignore + simcore_service_labels: SimcoreServiceLabels, + port: PortInt, + request_dns: str, + request_scheme: str, + request_simcore_user_agent: str, + can_save: bool, + run_id: RunID | None = None, + ) -> "SchedulerData": + # This constructor method sets current product + names_helper = DynamicSidecarNamesHelper.make(service.node_uuid) + + obj_dict = { + "service_name": names_helper.service_name_dynamic_sidecar, + "hostname": names_helper.service_name_dynamic_sidecar, + "port": port, + "node_uuid": service.node_uuid, + "project_id": service.project_id, + "user_id": service.user_id, + "key": service.key, + "version": service.version, + "service_resources": service.service_resources, + "product_name": service.product_name, + "paths_mapping": simcore_service_labels.paths_mapping, + "compose_spec": json.dumps(simcore_service_labels.compose_spec), + "container_http_entry": simcore_service_labels.container_http_entry, + "restart_policy": simcore_service_labels.restart_policy, + "dynamic_sidecar_network_name": names_helper.dynamic_sidecar_network_name, + "simcore_traefik_zone": names_helper.simcore_traefik_zone, + "request_dns": request_dns, + "request_scheme": request_scheme, + "proxy_service_name": names_helper.proxy_service_name, + "request_simcore_user_agent": request_simcore_user_agent, + "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}}, + } + if run_id: + obj_dict["run_id"] = run_id + return cls.parse_obj(obj_dict) + + @classmethod + def from_service_inspect( + cls, service_inspect: Mapping[str, Any] + ) -> "SchedulerData": + labels = service_inspect["Spec"]["Labels"] + return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) + + def as_label_data(self) -> str: + # compose_spec needs to be json encoded before encoding it to json + # and storing it in the label + return self.copy( + update={"compose_spec": json.dumps(self.compose_spec)}, deep=True + ).json() + + class Config: + extra = Extra.allow + allow_population_by_field_name = True diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py new file mode 100644 index 00000000000..a3bed72aa76 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py @@ -0,0 +1,153 @@ +from functools import cached_property +from pathlib import Path +from typing import Any, ClassVar + +from pydantic import BaseModel, Field + +from ..basic_regex import VERSION_RE +from ..basic_types import PortInt +from ..projects import ProjectID +from ..projects_nodes_io import NodeID +from ..services import DynamicServiceKey +from ..services_enums import ServiceBootType, ServiceState +from ..users import UserID + + +class CommonServiceDetails(BaseModel): + key: DynamicServiceKey = Field( + ..., + description="distinctive name for the node based on the docker registry path", + examples=[ + "simcore/services/dynamic/3dviewer", + ], + alias="service_key", + ) + version: str = Field( + ..., + description="semantic version number of the node", + regex=VERSION_RE, + examples=["1.0.0", "0.0.1"], + alias="service_version", + ) + + user_id: UserID + project_id: ProjectID + node_uuid: NodeID = Field(..., alias="service_uuid") + + +class ServiceDetails(CommonServiceDetails): + basepath: Path = Field( + default=None, + description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.", + alias="service_basepath", + ) + + class Config: + allow_population_by_field_name = True + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "key": "simcore/services/dynamic/3dviewer", + "version": "2.4.5", + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", + } + } + + +class RunningDynamicServiceDetails(ServiceDetails): + boot_type: ServiceBootType = Field( + default=ServiceBootType.V0, + description=( + "Describes how the dynamic services was started (legacy=V0, modern=V2)." + "Since legacy services do not have this label it defaults to V0." + ), + ) + + host: str = Field( + ..., description="the service swarm internal host name", alias="service_host" + ) + internal_port: PortInt = Field( + ..., description="the service swarm internal port", alias="service_port" + ) + published_port: PortInt = Field( + default=None, + description="the service swarm published port if any", + deprecated=True, + ) + + entry_point: str | None = Field( + default=None, + description="if empty the service entrypoint is on the root endpoint.", + deprecated=True, + ) + state: ServiceState = Field( + ..., description="service current state", alias="service_state" + ) + message: str | None = Field( + default=None, + description="additional information related to service state", + alias="service_message", + ) + + @cached_property + def legacy_service_url(self) -> str: + return f"http://{self.host}:{self.internal_port}{self.basepath}" # NOSONAR + + @classmethod + def from_scheduler_data( + cls, + node_uuid: NodeID, + scheduler_data: "SchedulerData", # type: ignore + service_state: ServiceState, + service_message: str, + ) -> "RunningDynamicServiceDetails": + return cls.parse_obj( + { + "boot_type": ServiceBootType.V2, + "user_id": scheduler_data.user_id, + "project_id": scheduler_data.project_id, + "service_uuid": node_uuid, + "service_key": scheduler_data.key, + "service_version": scheduler_data.version, + "service_host": scheduler_data.service_name, + "service_port": scheduler_data.service_port, + "service_state": service_state.value, + "service_message": service_message, + } + ) + + class Config(ServiceDetails.Config): + keep_untouched = (cached_property,) + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + "boot_type": "V0", + "key": "simcore/services/dynamic/3dviewer", + "version": "2.4.5", + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "host": "3dviewer_75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "internal_port": 8888, + "state": "running", + "message": "", + "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + }, + { + "boot_type": "V2", + "key": "simcore/services/dynamic/dy-static-file-viewer-dynamic-sidecar", + "version": "1.0.0", + "user_id": 234, + "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", + "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "host": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa", + "internal_port": 80, + "state": "running", + "message": "", + "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", + }, + ] + } diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/health.py b/packages/models-library/src/models_library/api_schemas_directorv2/health.py new file mode 100644 index 00000000000..1f578888b18 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/health.py @@ -0,0 +1,14 @@ +from typing import Any, ClassVar + +from pydantic import BaseModel + + +class HealthCheckGet(BaseModel): + timestamp: str + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "timestamp": "simcore_service_directorv2.api.routes.health@2023-07-03T12:59:12.024551+00:00" + } + } diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/meta.py b/packages/models-library/src/models_library/api_schemas_directorv2/meta.py new file mode 100644 index 00000000000..ef140792d38 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_directorv2/meta.py @@ -0,0 +1,27 @@ +import re +from typing import Any, ClassVar + +from pydantic import BaseModel, ConstrainedStr, Field + +from ..basic_regex import VERSION_RE + + +class VersionStr(ConstrainedStr): + regex = re.compile(VERSION_RE) + + +class Meta(BaseModel): + name: str + version: VersionStr + released: dict[str, VersionStr] | None = Field( + None, description="Maps every route's path tag with a released version" + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "name": "simcore_service_foo", + "version": "2.4.45", + "released": {"v1": "1.3.4", "v2": "2.4.45"}, + } + } From 383a6803825d89056500f4fc6055b998951bb6f0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:35:10 +0200 Subject: [PATCH 02/20] dv2 service src --- .../api/dependencies/dynamic_services.py | 4 +++- .../api/routes/clusters.py | 16 ++++++++-------- .../api/routes/dynamic_services.py | 17 +++++++++-------- .../api/routes/health.py | 2 +- .../api/routes/meta.py | 2 +- .../simcore_service_director_v2/cli/_core.py | 11 +++++------ .../core/settings.py | 5 +++-- .../modules/dask_client.py | 2 +- .../modules/db/repositories/clusters.py | 19 +++++++++---------- .../modules/director_v0.py | 4 +++- .../modules/dynamic_sidecar/_namespace.py | 5 +++-- .../dynamic_sidecar/api_client/_public.py | 4 +++- .../dynamic_sidecar/docker_api/_core.py | 16 ++++++++++------ .../dynamic_sidecar/docker_api/_volume.py | 4 +++- .../docker_service_specs/proxy.py | 4 +++- .../docker_service_specs/sidecar.py | 8 ++++++-- .../docker_service_specs/volume_remover.py | 4 +++- .../modules/dynamic_sidecar/docker_states.py | 7 ++++--- .../modules/dynamic_sidecar/scheduler/_abc.py | 11 +++++------ .../dynamic_sidecar/scheduler/_core/_abc.py | 5 +++-- .../scheduler/_core/_events.py | 13 +++++++------ .../scheduler/_core/_events_utils.py | 10 +++++----- .../scheduler/_core/_observer.py | 10 +++++----- .../scheduler/_core/_scheduler.py | 18 +++++++++--------- .../scheduler/_core/_scheduler_utils.py | 14 ++++++++------ .../dynamic_sidecar/scheduler/_task.py | 10 +++++----- 26 files changed, 125 insertions(+), 100 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/api/dependencies/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/dependencies/dynamic_services.py index 1f6914aa1e2..60dfc73c74e 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/dependencies/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/dependencies/dynamic_services.py @@ -1,12 +1,14 @@ import logging from fastapi import Depends, Request +from models_library.api_schemas_directorv2.dynamic_services import ( + RunningDynamicServiceDetails, +) from models_library.projects_nodes import NodeID from servicelib.logging_utils import log_decorator from starlette.datastructures import URL from ...core.settings import DynamicServicesSettings -from ...models.schemas.dynamic_services import RunningDynamicServiceDetails from ...modules.director_v0 import DirectorV0Client from ...modules.dynamic_services import ServicesClient from ...modules.dynamic_sidecar.scheduler import DynamicSidecarsScheduler diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/clusters.py b/services/director-v2/src/simcore_service_director_v2/api/routes/clusters.py index 1ff69155c09..0b8cde9d87e 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/clusters.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/clusters.py @@ -4,6 +4,14 @@ from aiocache import cached from fastapi import APIRouter, Depends, HTTPException +from models_library.api_schemas_directorv2.clusters import ( + ClusterCreate, + ClusterDetails, + ClusterDetailsGet, + ClusterGet, + ClusterPatch, + ClusterPing, +) from models_library.clusters import DEFAULT_CLUSTER_ID, Cluster, ClusterID from models_library.users import UserID from starlette import status @@ -14,14 +22,6 @@ DaskClientAcquisisitonError, ) from ...core.settings import ComputationalBackendSettings -from ...models.schemas.clusters import ( - ClusterCreate, - ClusterDetails, - ClusterDetailsGet, - ClusterGet, - ClusterPatch, - ClusterPing, -) from ...modules.dask_clients_pool import DaskClientsPool from ...modules.db.repositories.clusters import ClustersRepository from ...utils.dask_client_utils import test_scheduler_endpoint diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py index 0349179d762..371ba7cc387 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py @@ -1,10 +1,17 @@ import asyncio import logging -from typing import Coroutine, cast +from collections.abc import Coroutine +from typing import cast import httpx from fastapi import APIRouter, Depends, Header, HTTPException, Request from fastapi.responses import RedirectResponse +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceCreate, + DynamicServiceGet, + RetrieveDataIn, + RetrieveDataOutEnveloped, +) from models_library.projects import ProjectID from models_library.projects_nodes import NodeID from models_library.service_settings_labels import SimcoreServiceLabels @@ -25,12 +32,6 @@ from ...api.dependencies.database import get_repository from ...api.dependencies.rabbitmq import get_rabbitmq_client from ...core.settings import DynamicServicesSettings, DynamicSidecarSettings -from ...models.domains.dynamic_services import ( - DynamicServiceCreate, - DynamicServiceGet, - RetrieveDataIn, - RetrieveDataOutEnveloped, -) from ...modules import projects_networks from ...modules.db.repositories.projects import ProjectsRepository from ...modules.db.repositories.projects_networks import ProjectsNetworksRepository @@ -298,7 +299,7 @@ async def service_restart_containers( try: await scheduler.restart_containers(node_uuid) except DynamicSidecarNotFoundError as error: - raise LegacyServiceIsNotSupportedError() from error + raise LegacyServiceIsNotSupportedError from error return NoContentResponse() diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/health.py b/services/director-v2/src/simcore_service_director_v2/api/routes/health.py index 94078530e10..a4cf1f9619f 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/health.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/health.py @@ -1,7 +1,7 @@ import datetime from fastapi import APIRouter -from models_library.api_schemas_directorv2 import HealthCheckGet +from models_library.api_schemas_directorv2.health import HealthCheckGet router = APIRouter() diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/meta.py b/services/director-v2/src/simcore_service_director_v2/api/routes/meta.py index d0554b88ea2..b4471bc6fe7 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/meta.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/meta.py @@ -1,7 +1,7 @@ from fastapi import APIRouter +from models_library.api_schemas_directorv2.meta import Meta, VersionStr from ...meta import API_VERSION, API_VTAG -from ...models.schemas.meta import Meta, VersionStr router = APIRouter() diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_core.py b/services/director-v2/src/simcore_service_director_v2/cli/_core.py index 821b6a27033..c27277048dc 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_core.py @@ -7,9 +7,14 @@ import typer from fastapi import FastAPI, status from httpx import AsyncClient, HTTPError +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DynamicSidecarNamesHelper, +) from models_library.projects import NodeIDStr, ProjectID from models_library.projects_nodes_io import NodeID from models_library.services import ServiceType +from models_library.services_enums import ServiceBootType, ServiceState from pydantic import AnyHttpUrl, BaseModel, PositiveInt, parse_obj_as from rich.live import Live from rich.table import Table @@ -20,12 +25,6 @@ from ..core.application import create_base_app from ..core.settings import AppSettings -from ..models.domains.dynamic_services import DynamicServiceGet -from ..models.schemas.dynamic_services import ( - DynamicSidecarNamesHelper, - ServiceBootType, - ServiceState, -) from ..modules import db, director_v0, dynamic_sidecar from ..modules.db.repositories.projects import ProjectsRepository from ..modules.director_v0 import DirectorV0Client diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 1c14380dfb1..e1ca1784f15 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -9,6 +9,9 @@ from functools import cached_property from pathlib import Path +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_DOCKER_IMAGE_RE, +) from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -48,8 +51,6 @@ from simcore_postgres_database.models.clusters import ClusterType from simcore_sdk.node_ports_v2 import FileLinkType -from ..models.schemas.constants import DYNAMIC_SIDECAR_DOCKER_IMAGE_RE - logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index bf9e933777a..dbdf458e86d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -37,6 +37,7 @@ LogFileUploadURL, ) from fastapi import FastAPI +from models_library.api_schemas_directorv2.clusters import ClusterDetails, Scheduler from models_library.clusters import ClusterAuthentication, ClusterID from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -61,7 +62,6 @@ from ..core.settings import AppSettings, ComputationalBackendSettings from ..models.domains.comp_runs import MetadataDict from ..models.domains.comp_tasks import Image -from ..models.schemas.clusters import ClusterDetails, Scheduler from ..modules.storage import StorageClient from ..utils.dask import ( check_communication_with_scheduler_is_open, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py index 2dd8b048878..5724718a71a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py @@ -1,9 +1,10 @@ import logging -from typing import Iterable +from collections.abc import Iterable import psycopg2 import sqlalchemy as sa from aiopg.sa import connection +from models_library.api_schemas_directorv2.clusters import ClusterCreate, ClusterPatch from models_library.clusters import ( CLUSTER_ADMIN_RIGHTS, CLUSTER_MANAGER_RIGHTS, @@ -26,7 +27,6 @@ ClusterInvalidOperationError, ClusterNotFoundError, ) -from ....models.schemas.clusters import ClusterCreate, ClusterPatch from ....utils.db import to_clusters_db from ._base import BaseRepository @@ -101,11 +101,12 @@ async def _compute_user_access_rights( user_groups = await result.fetchall() # get the primary group first, as it has precedence - if primary_group_row := next( - filter(lambda ugrp: ugrp[1] == GroupType.PRIMARY, user_groups), None - ): - if primary_grp_rights := cluster.access_rights.get(primary_group_row.gid): - return primary_grp_rights + if ( + primary_group_row := next( + filter(lambda ugrp: ugrp[1] == GroupType.PRIMARY, user_groups), None + ) + ) and (primary_grp_rights := cluster.access_rights.get(primary_group_row.gid)): + return primary_grp_rights solved_rights = CLUSTER_NO_RIGHTS.dict() for group_row in filter(lambda ugrp: ugrp[1] != GroupType.PRIMARY, user_groups): @@ -263,9 +264,7 @@ async def update_cluster( # pylint: disable=too-many-branches clusters_list = await _clusters_from_cluster_ids(conn, {cluster_id}) if not clusters_list: raise ClusterNotFoundError(cluster_id=cluster_id) - the_cluster = clusters_list[0] - - return the_cluster + return clusters_list[0] async def delete_cluster(self, user_id: UserID, cluster_id: ClusterID) -> None: async with self.db_engine.acquire() as conn: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py index 619c518fd67..ade533a413e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py @@ -10,6 +10,9 @@ import httpx import yarl from fastapi import FastAPI, HTTPException, status +from models_library.api_schemas_directorv2.dynamic_services import ( + RunningDynamicServiceDetails, +) from models_library.projects import ProjectID from models_library.projects_nodes import NodeID from models_library.service_settings_labels import SimcoreServiceLabels @@ -18,7 +21,6 @@ from servicelib.logging_utils import log_decorator from ..core.settings import DirectorV0Settings -from ..models.schemas.dynamic_services import RunningDynamicServiceDetails from ..models.schemas.services import ServiceExtras from ..utils.client_decorators import handle_errors, handle_retry from ..utils.clients import unenvelope_or_raise_error diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py index caa43a669d6..154cd9a0600 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py @@ -1,7 +1,8 @@ +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) from models_library.projects_nodes import NodeID -from ...models.schemas.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX - def get_compose_namespace(node_uuid: NodeID) -> str: # To avoid collisions for started docker resources a unique identifier is computed: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 3511827292e..322593c9e76 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -5,6 +5,9 @@ from fastapi import FastAPI, status from httpx import AsyncClient +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -23,7 +26,6 @@ from servicelib.utils import logged_gather from ....core.settings import DynamicSidecarSettings -from ....models.schemas.dynamic_services import SchedulerData from ....modules.dynamic_sidecar.docker_api import get_or_create_networks_ids from ..errors import EntrypointContainerNotFoundError from ._errors import BaseClientHTTPError, UnexpectedStatusError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index 6d5e133cdf7..ce6ccdc1e1b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -8,9 +8,19 @@ from aiodocker.utils import clean_filters, clean_map from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + NetworkId, + SchedulerData, + ServiceId, +) from models_library.docker import to_simcore_runtime_docker_label_key from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID +from models_library.services_enums import ServiceState from servicelib.json_serialization import json_dumps from servicelib.utils import logged_gather from starlette import status @@ -21,12 +31,6 @@ from tenacity.wait import wait_exponential, wait_random_exponential from ....core.settings import DynamicSidecarSettings -from ....models.schemas.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) -from ....models.schemas.dynamic_services import SchedulerData, ServiceId, ServiceState -from ....models.schemas.dynamic_services.scheduler import NetworkId from ....utils.dict_utils import get_leaf_key_paths, nested_update from ..docker_states import TASK_STATES_RUNNING, extract_task_state from ..errors import DockerServiceNotFoundError, DynamicSidecarError, GenericDockerError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py index 749b71256ad..59d21c90471 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py @@ -2,6 +2,9 @@ from datetime import datetime, timezone from fastapi.encoders import jsonable_encoder +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_VOLUME_REMOVER_PREFIX, +) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID @@ -14,7 +17,6 @@ from tenacity.wait import wait_fixed from ....core.settings import DynamicSidecarSettings -from ....models.schemas.constants import DYNAMIC_VOLUME_REMOVER_PREFIX from ..docker_service_specs.volume_remover import spec_volume_removal_service from ._utils import docker_client diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py index 6f9d90542c3..3639edd6d99 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py @@ -1,5 +1,8 @@ from typing import Any +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.docker import StandardSimcoreDockerLabels from models_library.services_resources import ( CPU_10_PERCENT, @@ -10,7 +13,6 @@ from pydantic import ByteSize from ....core.settings import DynamicSidecarProxySettings, DynamicSidecarSettings -from ....models.schemas.dynamic_services import SchedulerData from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index e7e416380b1..1e090bde362 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -2,6 +2,12 @@ from copy import deepcopy from models_library.aiodocker_api import AioDockerServiceSpec +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, +) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import ( StandardSimcoreDockerLabels, @@ -12,8 +18,6 @@ from servicelib.json_serialization import json_dumps from ....core.settings import AppSettings, DynamicSidecarSettings -from ....models.schemas.constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL -from ....models.schemas.dynamic_services import SchedulerData from .._namespace import get_compose_namespace from ..volumes import DynamicSidecarVolumesPathsResolver from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py index 4f086424e75..9d7b80c3cef 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py @@ -5,6 +5,9 @@ from uuid import uuid4 from models_library.aiodocker_api import AioDockerServiceSpec +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_VOLUME_REMOVER_PREFIX, +) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services_resources import ( @@ -17,7 +20,6 @@ from pydantic import parse_obj_as from ....core.settings import DynamicSidecarSettings -from ....models.schemas.constants import DYNAMIC_VOLUME_REMOVER_PREFIX class DockerVersion(str): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py index fca9f89acf7..6af5120b401 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py @@ -3,10 +3,11 @@ """ import logging +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, +) from models_library.generated_models.docker_rest_api import ContainerState - -from ...models.schemas.dynamic_services import ServiceState -from ...models.schemas.dynamic_services.scheduler import DockerContainerInspect +from models_library.services_enums import ServiceState logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py index 975021f445b..4eb3cf5a84c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py @@ -1,5 +1,10 @@ from abc import ABC, abstractmethod +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceCreate, + RetrieveDataOutEnveloped, + RunningDynamicServiceDetails, +) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -9,12 +14,6 @@ from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress -from ....models.domains.dynamic_services import ( - DynamicServiceCreate, - RetrieveDataOutEnveloped, -) -from ....models.schemas.dynamic_services import RunningDynamicServiceDetails - class SchedulerInternalsInterface(ABC): @abstractmethod diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py index 98394de69da..c3a50abcd48 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py @@ -3,8 +3,9 @@ from abc import ABC, abstractmethod from fastapi import FastAPI - -from .....models.schemas.dynamic_services import SchedulerData +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) class DynamicSchedulerEvent(ABC): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index ff85e690e4e..28ab6566843 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -6,6 +6,13 @@ from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + DynamicSidecarStatus, + NetworkId, + SchedulerData, +) from models_library.projects import ProjectAtDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeIDStr @@ -30,12 +37,6 @@ from tenacity.wait import wait_fixed from .....core.settings import DynamicSidecarProxySettings, DynamicSidecarSettings -from .....models.schemas.dynamic_services import DynamicSidecarStatus, SchedulerData -from .....models.schemas.dynamic_services.scheduler import ( - DockerContainerInspect, - DockerStatus, - NetworkId, -) from .....utils.db import get_repository from .....utils.dict_utils import nested_update from ....catalog import CatalogClient diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index ad65f11af82..1b637e3ede6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -6,6 +6,11 @@ from typing import Any, Deque, Final from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + SchedulerData, +) from models_library.projects_networks import ProjectsNetworks from models_library.projects_nodes import NodeID from models_library.projects_nodes_io import NodeIDStr @@ -30,11 +35,6 @@ from .....core.errors import NodeRightsAcquireError from .....core.settings import AppSettings, DynamicSidecarSettings -from .....models.schemas.dynamic_services.scheduler import ( - DockerContainerInspect, - DockerStatus, - SchedulerData, -) from .....utils.db import get_repository from ....db.repositories.projects import ProjectsRepository from ....db.repositories.projects_networks import ProjectsNetworksRepository diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index ef34423d7c9..96a3948eb56 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -6,6 +6,11 @@ from math import floor from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DynamicSidecarStatus, + SchedulerData, + ServiceName, +) from servicelib.error_codes import create_error_code from .....core.settings import ( @@ -13,11 +18,6 @@ DynamicServicesSettings, DynamicSidecarSettings, ) -from .....models.schemas.dynamic_services import ( - DynamicSidecarStatus, - SchedulerData, - ServiceName, -) from ...docker_api import ( are_sidecar_and_proxy_services_present, is_dynamic_sidecar_stack_missing, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index fe9358a0f59..6af778d01c5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -21,6 +21,15 @@ from dataclasses import dataclass, field from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceCreate, + RetrieveDataOutEnveloped, + RunningDynamicServiceDetails, +) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, + ServiceName, +) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -33,15 +42,6 @@ from servicelib.fastapi.long_running_tasks.server import TaskProgress from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings -from .....models.domains.dynamic_services import ( - DynamicServiceCreate, - RetrieveDataOutEnveloped, -) -from .....models.schemas.dynamic_services import ( - RunningDynamicServiceDetails, - SchedulerData, - ServiceName, -) from ...api_client import SidecarsClient, get_sidecars_client from ...docker_api import update_scheduler_data_label from ...errors import DynamicSidecarError, DynamicSidecarNotFoundError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index 4143dfa48db..80ca6697d26 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -3,16 +3,18 @@ from typing import Final from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DynamicSidecarStatus, + SchedulerData, +) +from models_library.api_schemas_directorv2.dynamic_services_service import ( + RunningDynamicServiceDetails, +) from models_library.projects_nodes_io import NodeID +from models_library.services_enums import ServiceState from servicelib.fastapi.long_running_tasks.client import ProgressCallback from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings -from .....models.schemas.dynamic_services import ( - DynamicSidecarStatus, - RunningDynamicServiceDetails, - SchedulerData, - ServiceState, -) from ...api_client import SidecarsClient, get_sidecars_client from ...docker_api import ( get_dynamic_sidecar_state, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py index 543b32c2ce5..69c0906e0ee 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_task.py @@ -1,6 +1,11 @@ import logging from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceCreate, + RetrieveDataOutEnveloped, + RunningDynamicServiceDetails, +) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -11,11 +16,6 @@ from servicelib.fastapi.long_running_tasks.server import TaskProgress from ....core.settings import DynamicServicesSchedulerSettings -from ....models.domains.dynamic_services import ( - DynamicServiceCreate, - RetrieveDataOutEnveloped, -) -from ....models.schemas.dynamic_services import RunningDynamicServiceDetails from ._abc import SchedulerInternalsInterface, SchedulerPublicInterface from ._core._scheduler import Scheduler From 8ae47010587702c310c3ed1fda96870acf5dbc70 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:39:07 +0200 Subject: [PATCH 03/20] dv2 service tests --- ...t_dynamic_sidecar_nodeports_integration.py | 6 +-- .../director-v2/tests/integration/02/utils.py | 14 +++---- services/director-v2/tests/unit/conftest.py | 14 +++---- .../unit/test_api_route_dynamic_scheduler.py | 10 ++--- .../tests/unit/test_models_clusters.py | 4 +- .../unit/test_models_dynamic_services.py | 6 +-- ...ic_sidecar_docker_service_specs_sidecar.py | 7 ++-- .../test_modules_dynamic_sidecar_scheduler.py | 13 +++--- ...s_dynamic_sidecar_scheduler_core_events.py | 21 +++++----- ..._modules_dynamic_sidecar_scheduler_task.py | 21 +++++----- ...test_schemas_dynamic_services_scheduler.py | 4 +- .../unit/with_dbs/test_api_route_clusters.py | 22 +++++----- .../test_api_route_clusters_details.py | 10 ++--- .../test_api_route_dynamic_services.py | 14 +++---- .../tests/unit/with_dbs/test_cli.py | 10 ++--- ...test_modules_dynamic_sidecar_docker_api.py | 42 +++++++++---------- 16 files changed, 106 insertions(+), 112 deletions(-) diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 4ea874b414e..2c3b764aca8 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -37,6 +37,9 @@ assert_and_wait_for_pipeline_status, assert_computation_task_out_obj, ) +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) from models_library.clusters import DEFAULT_CLUSTER_ID from models_library.projects import ( Node, @@ -77,9 +80,6 @@ from simcore_sdk.node_ports_v2 import DBManager, Nodeports, Port from simcore_service_director_v2.core.settings import AppSettings, RCloneSettings from simcore_service_director_v2.models.schemas.comp_tasks import ComputationGet -from simcore_service_director_v2.models.schemas.constants import ( - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) from sqlalchemy.dialects.postgresql import insert as pg_insert from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 9e036b4acfb..68cd2e7f2c2 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -12,6 +12,13 @@ import aiodocker import httpx from fastapi import FastAPI +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_PROXY_SERVICE_PREFIX, + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.basic_types import PortInt from models_library.projects import Node, NodesDict from models_library.projects_nodes_io import NodeID @@ -28,13 +35,6 @@ X_SIMCORE_USER_AGENT, ) from simcore_service_director_v2.core.settings import DynamicSidecarSettings -from simcore_service_director_v2.models.schemas.constants import ( - DYNAMIC_PROXY_SERVICE_PREFIX, - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - SchedulerData, -) from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( DynamicSidecarsScheduler, ) diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index 7f35704084e..26c63f6abce 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -5,8 +5,7 @@ import logging import random import urllib.parse -from typing import ( - Any, +from collections.abc import ( AsyncIterable, AsyncIterator, Callable, @@ -14,6 +13,7 @@ Iterator, Mapping, ) +from typing import Any from unittest import mock import aiodocker @@ -27,6 +27,10 @@ from distributed.deploy.spec import SpecCluster from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, +) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.basic_types import PortInt from models_library.clusters import ClusterID from models_library.generated_models.docker_rest_api import ( @@ -41,12 +45,6 @@ from settings_library.s3 import S3Settings from simcore_sdk.node_ports_v2 import FileLinkType from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.domains.dynamic_services import ( - DynamicServiceCreate, -) -from simcore_service_director_v2.models.schemas.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, -) from simcore_service_director_v2.models.schemas.dynamic_services import ( SchedulerData, ServiceDetails, diff --git a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py index cdef0b84064..66d3b694af0 100644 --- a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py +++ b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py @@ -9,15 +9,13 @@ import respx from fastapi import status from httpx import Response +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from simcore_service_director_v2.models.domains.dynamic_services import ( - DynamicServiceCreate, -) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - SchedulerData, -) from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarNotFoundError, ) diff --git a/services/director-v2/tests/unit/test_models_clusters.py b/services/director-v2/tests/unit/test_models_clusters.py index 76f8801610e..8628d76d706 100644 --- a/services/director-v2/tests/unit/test_models_clusters.py +++ b/services/director-v2/tests/unit/test_models_clusters.py @@ -3,8 +3,7 @@ import pytest from faker import Faker -from pydantic import BaseModel, parse_obj_as -from simcore_service_director_v2.models.schemas.clusters import ( +from models_library.api_schemas_directorv2.clusters import ( AvailableResources, ClusterCreate, ClusterPatch, @@ -13,6 +12,7 @@ Worker, WorkerMetrics, ) +from pydantic import BaseModel, parse_obj_as @pytest.mark.parametrize( diff --git a/services/director-v2/tests/unit/test_models_dynamic_services.py b/services/director-v2/tests/unit/test_models_dynamic_services.py index 03c9084f24b..633301a4e18 100644 --- a/services/director-v2/tests/unit/test_models_dynamic_services.py +++ b/services/director-v2/tests/unit/test_models_dynamic_services.py @@ -3,15 +3,15 @@ from collections import namedtuple import pytest +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, +) from simcore_service_director_v2.models.schemas.dynamic_services import ( RunningDynamicServiceDetails, SchedulerData, ServiceBootType, ServiceState, ) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - DockerContainerInspect, -) from simcore_service_director_v2.modules.dynamic_sidecar.docker_states import ( CONTAINER_STATUSES_UNEXPECTED, extract_containers_minimum_statuses, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 50811f91b69..9bc4d8960ee 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -4,11 +4,10 @@ from typing import Any -from pytest import MonkeyPatch -from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( SchedulerData, ) +from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs.sidecar import ( _get_environment_variables, ) @@ -59,7 +58,7 @@ def test_dynamic_sidecar_env_vars( - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, scheduler_data_from_http_request: SchedulerData, project_env_devel_environment: dict[str, Any], ): diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 5eccb3ff9e6..a35a3abea1f 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -14,8 +14,10 @@ import respx from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, +) from models_library.service_settings_labels import SimcoreServiceLabels -from pytest import LogCaptureFixture, MonkeyPatch from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter @@ -26,9 +28,6 @@ SchedulerData, ServiceState, ) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - DockerContainerInspect, -) from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarError, DynamicSidecarNotFoundError, @@ -117,7 +116,7 @@ def mock_env( disable_postgres: None, disable_rabbitmq: None, mock_env: EnvVarsDict, - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, simcore_services_network_name: str, mock_docker_api: None, ) -> None: @@ -223,7 +222,7 @@ def mock_update_label(mocker: MockerFixture) -> Iterator[None]: @pytest.fixture -def mock_max_status_api_duration(monkeypatch: MonkeyPatch) -> Iterator[None]: +def mock_max_status_api_duration(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setenv("DYNAMIC_SIDECAR_STATUS_API_TIMEOUT_S", "0.0001") yield @@ -451,7 +450,7 @@ def mocked_app() -> AsyncMock: async def test_regression_remove_service_from_observation( mocked_app: AsyncMock, faker: Faker, - caplog_debug_level: LogCaptureFixture, + caplog_debug_level: pytest.LogCaptureFixture, missing_to_observe_entry: bool, ): scheduler = Scheduler(mocked_app) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py index 27d8ce2811a..f52311311f9 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py @@ -7,18 +7,17 @@ import pytest from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + ContainerState, + DockerContainerInspect, + DockerStatus, +) from pydantic import PositiveFloat, PositiveInt -from pytest import LogCaptureFixture, MonkeyPatch from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from servicelib.exception_utils import _SKIPS_MESSAGE from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - ContainerState, - DockerContainerInspect, - DockerStatus, -) from simcore_service_director_v2.modules.dynamic_sidecar.api_client import ( BaseClientHTTPError, ) @@ -34,7 +33,7 @@ def mock_env( disable_postgres: None, mock_env: EnvVarsDict, - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, ) -> None: setenvs_from_dict( monkeypatch, @@ -93,7 +92,9 @@ def scheduler_data( @pytest.fixture() -def caplog_debug(caplog: LogCaptureFixture) -> Iterable[LogCaptureFixture]: +def caplog_debug( + caplog: pytest.LogCaptureFixture, +) -> Iterable[pytest.LogCaptureFixture]: with caplog.at_level( logging.DEBUG, ): @@ -104,7 +105,7 @@ async def test_event_get_status_network_connectivity( mock_sidecars_client_always_fail: None, minimal_app: FastAPI, scheduler_data: SchedulerData, - caplog_debug: LogCaptureFixture, + caplog_debug: pytest.LogCaptureFixture, ): caplog_debug.clear() with pytest.raises(BaseClientHTTPError): @@ -119,7 +120,7 @@ async def test_event_get_status_recovers_after_error( mock_sidecars_client_stops_failing: None, minimal_app: FastAPI, scheduler_data: SchedulerData, - caplog_debug: LogCaptureFixture, + caplog_debug: pytest.LogCaptureFixture, ): caplog_debug.clear() for _ in range(REPEAT_COUNT): diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py index c76b49a0377..d9985b7b80d 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py @@ -4,22 +4,22 @@ import asyncio import re +from collections.abc import Iterator from dataclasses import dataclass -from typing import Final, Iterator +from typing import Final from unittest.mock import AsyncMock import httpx import pytest import respx from fastapi import FastAPI -from pytest import FixtureRequest, MonkeyPatch +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from respx.router import MockRouter -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - SchedulerData, -) from simcore_service_director_v2.modules.dynamic_sidecar.api_client._public import ( SidecarsClient, ) @@ -45,7 +45,7 @@ def mock_env( disable_postgres: None, disable_rabbitmq: None, mock_env: EnvVarsDict, - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, simcore_services_network_name: str, docker_swarm: None, mock_docker_api: None, @@ -88,7 +88,7 @@ def mock_containers_docker_status( name="containers_docker_status", ).mock(httpx.Response(200, json={})) mock.get(f"{service_endpoint}/health", name="is_healthy").respond( - json=dict(is_healthy=True) + json={"is_healthy": True} ) yield mock @@ -131,7 +131,7 @@ def increment(self) -> None: @pytest.fixture(params=[True, False]) -def error_raised_by_saving_state(request: FixtureRequest) -> bool: +def error_raised_by_saving_state(request: pytest.FixtureRequest) -> bool: return request.param # type: ignore @@ -194,7 +194,8 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = ( use_case.wait_for_manual_intervention_after_error ) - raise RuntimeError("Failed as planned") + msg = "Failed as planned" + raise RuntimeError(msg) test_defined_scheduler_events: list[type[DynamicSchedulerEvent]] = [ AlwaysTriggersDynamicSchedulerEvent @@ -214,7 +215,7 @@ def mock_remove_calls(mocker: MockerFixture) -> None: @pytest.fixture(params=[True, False]) -def node_present_in_db(request: FixtureRequest) -> bool: +def node_present_in_db(request: pytest.FixtureRequest) -> bool: return request.param diff --git a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py index eab2f93727b..68e34b7a783 100644 --- a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py +++ b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py @@ -1,12 +1,12 @@ # pylint: disable=redefined-outer-name +from collections.abc import Iterator from contextlib import contextmanager from copy import deepcopy from pathlib import Path -from typing import Iterator import pytest -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( SchedulerData, ) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py index 306db6cb916..f050f6ca155 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py @@ -4,7 +4,8 @@ import json import random -from typing import Any, Callable, Iterator +from collections.abc import Callable, Iterator +from typing import Any import httpx import pytest @@ -13,6 +14,12 @@ from distributed.deploy.spec import SpecCluster from faker import Faker from httpx import URL +from models_library.api_schemas_directorv2.clusters import ( + ClusterCreate, + ClusterGet, + ClusterPatch, + ClusterPing, +) from models_library.clusters import ( CLUSTER_ADMIN_RIGHTS, CLUSTER_MANAGER_RIGHTS, @@ -24,16 +31,9 @@ SimpleAuthentication, ) from pydantic import AnyHttpUrl, SecretStr, parse_obj_as -from pytest import MonkeyPatch from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.utils_cli import create_json_encoder_wo_secrets from simcore_postgres_database.models.clusters import ClusterType, clusters -from simcore_service_director_v2.models.schemas.clusters import ( - ClusterCreate, - ClusterGet, - ClusterPatch, - ClusterPing, -) from starlette import status pytest_simcore_core_services_selection = [ @@ -49,7 +49,7 @@ def clusters_config( mock_env: EnvVarsDict, postgres_db: sa.engine.Engine, postgres_host_config: dict[str, str], - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, dask_spec_local_cluster: SpecCluster, ): monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "1") @@ -335,7 +335,7 @@ async def test_create_cluster( # let's check that DB is correctly setup, there is one entry with postgres_db.connect() as conn: - cluster_entry = conn.execute( + conn.execute( sa.select(clusters).where(clusters.c.name == cluster_data.name) ).one() @@ -544,7 +544,7 @@ async def test_update_another_cluster( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - original_cluster = parse_obj_as(ClusterGet, response.json()) + parse_obj_as(ClusterGet, response.json()) # let's try to modify stuff as we are user 2 for cluster_patch in [ diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py index e900f6dc257..70c750ab541 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py @@ -3,7 +3,8 @@ # pylint:disable=redefined-outer-name import json -from typing import Any, AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable +from typing import Any import httpx import pytest @@ -13,12 +14,11 @@ from distributed import Client as DaskClient from distributed.deploy.spec import SpecCluster from faker import Faker +from models_library.api_schemas_directorv2.clusters import ClusterDetailsGet from models_library.clusters import Cluster, ClusterID, SimpleAuthentication from models_library.users import UserID from pydantic import SecretStr -from pytest import MonkeyPatch from pytest_simcore.helpers.typing_env import EnvVarsDict -from simcore_service_director_v2.models.schemas.clusters import ClusterDetailsGet from starlette import status from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay @@ -37,7 +37,7 @@ def clusters_config( mock_env: EnvVarsDict, postgres_db: sa.engine.Engine, postgres_host_config: dict[str, str], - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, dask_spec_local_cluster: SpecCluster, ): monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "1") @@ -259,7 +259,7 @@ def do_some_work(x: int): # let's wait for the result result = task.result(timeout=_TASK_SLEEP_TIME + 5) assert result - assert await result == True + assert await result is True # wait for the computation to effectively stop async for attempt in AsyncRetrying( reraise=True, stop=stop_after_delay(60), wait=wait_fixed(1) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index 83ec4b1f345..2921d8489f6 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -15,6 +15,13 @@ import respx from fastapi import FastAPI from httpx import URL, QueryParams +from models_library.api_schemas_directorv2.dynamic_services import ( + DynamicServiceCreate, + RetrieveDataOutEnveloped, +) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + SchedulerData, +) from models_library.projects_nodes_io import NodeID from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture @@ -25,16 +32,9 @@ X_DYNAMIC_SIDECAR_REQUEST_SCHEME, X_SIMCORE_USER_AGENT, ) -from simcore_service_director_v2.models.domains.dynamic_services import ( - DynamicServiceCreate, - RetrieveDataOutEnveloped, -) from simcore_service_director_v2.models.schemas.dynamic_services import ( RunningDynamicServiceDetails, ) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - SchedulerData, -) from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarNotFoundError, ) diff --git a/services/director-v2/tests/unit/with_dbs/test_cli.py b/services/director-v2/tests/unit/with_dbs/test_cli.py index 5fe28a60059..55420ab64d1 100644 --- a/services/director-v2/tests/unit/with_dbs/test_cli.py +++ b/services/director-v2/tests/unit/with_dbs/test_cli.py @@ -4,14 +4,16 @@ import os import re import traceback +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from typing import Any, AsyncIterable, AsyncIterator, Awaitable, Callable +from typing import Any import pytest import respx from click.testing import Result from faker import Faker from fastapi import status +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.projects import ProjectAtDB from models_library.projects_nodes_io import NodeID from pytest_mock.plugin import MockerFixture @@ -21,9 +23,6 @@ from simcore_service_director_v2.cli._close_and_save_service import ( ThinDV2LocalhostClient, ) -from simcore_service_director_v2.models.domains.dynamic_services import ( - DynamicServiceGet, -) from simcore_service_director_v2.models.schemas.dynamic_services import ( RunningDynamicServiceDetails, ) @@ -87,7 +86,8 @@ def mock_save_service_state(mocker: MockerFixture) -> None: @pytest.fixture def mock_save_service_state_as_failing(mocker: MockerFixture) -> None: async def _always_raise(*args, **kwargs) -> None: - raise Exception("I AM FAILING NOW") # pylint: disable=broad-exception-raised + msg = "I AM FAILING NOW" + raise Exception(msg) # pylint: disable=broad-exception-raised mocker.patch( "simcore_service_director_v2.modules.dynamic_sidecar.api_client._public.SidecarsClient.save_service_state", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index 8769cd4bdf4..721abb5de8e 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -3,10 +3,12 @@ # pylint: disable=protected-access import asyncio +import contextlib import datetime import logging import sys -from typing import Any, AsyncIterable, AsyncIterator +from collections.abc import AsyncIterable, AsyncIterator +from typing import Any from uuid import UUID, uuid4 import aiodocker @@ -15,26 +17,25 @@ from aiodocker.volumes import DockerVolume from faker import Faker from fastapi.encoders import jsonable_encoder -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.users import UserID -from pytest import FixtureRequest, MonkeyPatch -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_director_v2.core.settings import DynamicSidecarSettings -from simcore_service_director_v2.models.schemas.constants import ( +from models_library.api_schemas_directorv2.constants import ( DYNAMIC_PROXY_SERVICE_PREFIX, DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, DYNAMIC_SIDECAR_SERVICE_PREFIX, DYNAMIC_VOLUME_REMOVER_PREFIX, ) +from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( + DockerContainerInspect, + SimcoreServiceLabels, +) +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from models_library.users import UserID +from pytest_simcore.helpers.utils_envs import EnvVarsDict +from simcore_service_director_v2.core.settings import DynamicSidecarSettings from simcore_service_director_v2.models.schemas.dynamic_services import ( SchedulerData, ServiceState, ) -from simcore_service_director_v2.models.schemas.dynamic_services.scheduler import ( - DockerContainerInspect, - SimcoreServiceLabels, -) from simcore_service_director_v2.modules.dynamic_sidecar import docker_api from simcore_service_director_v2.modules.dynamic_sidecar.docker_api._core import ( _update_service_spec, @@ -70,7 +71,7 @@ @pytest.fixture def dynamic_sidecar_settings( - monkeypatch: MonkeyPatch, mock_env: EnvVarsDict + monkeypatch: pytest.MonkeyPatch, mock_env: EnvVarsDict ) -> DynamicSidecarSettings: monkeypatch.setenv("DYNAMIC_SIDECAR_IMAGE", "local/dynamic-sidecar:MOCKED") monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "false") @@ -328,13 +329,13 @@ def service_name() -> str: for example in SimcoreServiceLabels.Config.schema_extra["examples"] ], ) -def labels_example(request: FixtureRequest) -> SimcoreServiceLabels: +def labels_example(request: pytest.FixtureRequest) -> SimcoreServiceLabels: return request.param @pytest.fixture(params=[None, datetime.datetime.now(tz=datetime.timezone.utc)]) def time_dy_sidecar_became_unreachable( - request: FixtureRequest, + request: pytest.FixtureRequest, ) -> datetime.datetime | None: return request.param @@ -382,7 +383,7 @@ async def mock_service( ) def test_settings__valid_network_names( simcore_services_network_name: str, - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, dynamic_sidecar_settings: DynamicSidecarSettings, ) -> None: items = dynamic_sidecar_settings.dict() @@ -750,8 +751,7 @@ async def test_regression_update_service_update_out_of_sequence( async def target_node_id(async_docker_client: aiodocker.Docker) -> str: # get a node's ID docker_nodes = await async_docker_client.nodes.list() - target_node_id = docker_nodes[0]["ID"] - return target_node_id + return docker_nodes[0]["ID"] async def test_constrain_service_to_node( @@ -793,10 +793,8 @@ async def named_volumes( # remove volume if still present for named_volume in named_volumes: - try: + with contextlib.suppress(aiodocker.DockerError): await named_volume.delete() - except aiodocker.DockerError: - pass async def is_volume_present( @@ -874,7 +872,7 @@ def volume_removal_services_names(faker: Faker) -> set[str]: @pytest.fixture(params=[0, 2]) -def service_timeout_s(request: FixtureRequest) -> int: +def service_timeout_s(request: pytest.FixtureRequest) -> int: return request.param # type: ignore From 2f2bcd5c2718b2d855b7a044929ecdb3b802e4b6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:40:13 +0200 Subject: [PATCH 04/20] minor models_library --- .../src/models_library/clusters.py | 17 ++++++++--------- .../models-library/src/models_library/groups.py | 3 ++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 146773e74f6..8f36790457c 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -1,7 +1,6 @@ from enum import auto -from typing import Any, ClassVar, Final, Literal, TypeAlias, Union +from typing import Any, ClassVar, Final, Literal, TypeAlias -from models_library.utils.common_validators import create_enums_pre_validator from pydantic import ( AnyUrl, BaseModel, @@ -15,6 +14,7 @@ from pydantic.types import NonNegativeInt from .users import GroupID +from .utils.common_validators import create_enums_pre_validator from .utils.enums import StrAutoEnum @@ -95,13 +95,12 @@ class NoAuthentication(BaseAuthentication): InternalClusterAuthentication: TypeAlias = NoAuthentication -ExternalClusterAuthentication: TypeAlias = Union[ - SimpleAuthentication, KerberosAuthentication, JupyterHubTokenAuthentication -] -ClusterAuthentication: TypeAlias = Union[ - ExternalClusterAuthentication, - InternalClusterAuthentication, -] +ExternalClusterAuthentication: TypeAlias = ( + SimpleAuthentication | KerberosAuthentication | JupyterHubTokenAuthentication +) +ClusterAuthentication: TypeAlias = ( + ExternalClusterAuthentication | InternalClusterAuthentication +) class BaseCluster(BaseModel): diff --git a/packages/models-library/src/models_library/groups.py b/packages/models-library/src/models_library/groups.py index e5ae95fc1aa..7305e2ce412 100644 --- a/packages/models-library/src/models_library/groups.py +++ b/packages/models-library/src/models_library/groups.py @@ -1,10 +1,11 @@ import enum from typing import Any, ClassVar, Final -from models_library.utils.common_validators import create_enums_pre_validator from pydantic import BaseModel, Field, validator from pydantic.types import PositiveInt +from .utils.common_validators import create_enums_pre_validator + EVERYONE_GROUP_ID: Final[int] = 1 From c7352b9eef2e1ca479c16197db35cf23132a56c7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:46:56 +0200 Subject: [PATCH 05/20] rm dyservcies models from dv2 --- .../models/domains/dynamic_services.py | 62 --- .../models/schemas/clusters.py | 202 -------- .../models/schemas/constants.py | 22 - .../schemas/dynamic_services/__init__.py | 2 - .../schemas/dynamic_services/scheduler.py | 476 ------------------ .../schemas/dynamic_services/service.py | 191 ------- .../models/schemas/meta.py | 25 - 7 files changed, 980 deletions(-) delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_services.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/constants.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/__init__.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/service.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/meta.py diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_services.py deleted file mode 100644 index f4c78e32c3c..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/domains/dynamic_services.py +++ /dev/null @@ -1,62 +0,0 @@ -from models_library.services import ServicePortKey -from models_library.services_resources import ( - ServiceResourcesDict, - ServiceResourcesDictHelpers, -) -from pydantic import BaseModel, ByteSize, Field - -from ..schemas.dynamic_services import RunningDynamicServiceDetails, ServiceDetails - - -class RetrieveDataIn(BaseModel): - port_keys: list[ServicePortKey] = Field( - ..., description="The port keys to retrieve data from" - ) - - -class RetrieveDataOut(BaseModel): - size_bytes: ByteSize = Field( - ..., description="The amount of data transferred by the retrieve call" - ) - - -class RetrieveDataOutEnveloped(BaseModel): - data: RetrieveDataOut - - @classmethod - def from_transferred_bytes( - cls, transferred_bytes: int - ) -> "RetrieveDataOutEnveloped": - return cls(data=RetrieveDataOut(size_bytes=ByteSize(transferred_bytes))) - - class Config: - schema_extra = {"examples": [{"data": {"size_bytes": 42}}]} - - -class DynamicServiceCreate(ServiceDetails): - service_resources: ServiceResourcesDict - - product_name: str = Field(..., description="Current product name") - can_save: bool = Field( - ..., description="the service data must be saved when closing" - ) - - class Config: - schema_extra = { - "example": { - "key": "simcore/services/dynamic/3dviewer", - "version": "2.4.5", - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "product_name": "osparc", - "can_save": True, - "service_resources": ServiceResourcesDictHelpers.Config.schema_extra[ - "examples" - ][0], - } - } - - -DynamicServiceGet = RunningDynamicServiceDetails diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py deleted file mode 100644 index 58d875dd1ab..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py +++ /dev/null @@ -1,202 +0,0 @@ -from typing import TypeAlias - -from models_library.clusters import ( - CLUSTER_ADMIN_RIGHTS, - CLUSTER_MANAGER_RIGHTS, - CLUSTER_USER_RIGHTS, - BaseCluster, - Cluster, - ClusterAccessRights, - ClusterAuthentication, - ClusterTypeInModel, - ExternalClusterAuthentication, -) -from models_library.generics import DictModel -from models_library.users import GroupID -from pydantic import ( - AnyHttpUrl, - BaseModel, - Field, - HttpUrl, - NonNegativeFloat, - root_validator, - validator, -) -from pydantic.networks import AnyUrl -from pydantic.types import ByteSize, PositiveFloat - - -class TaskCounts(BaseModel): - error: int = 0 - memory: int = 0 - executing: int = 0 - - -class WorkerMetrics(BaseModel): - cpu: float = Field(..., description="consumed % of cpus") - memory: ByteSize = Field(..., description="consumed memory") - num_fds: int = Field(..., description="consumed file descriptors") - task_counts: TaskCounts = Field(..., description="task details") - - -AvailableResources: TypeAlias = DictModel[str, PositiveFloat] - - -class UsedResources(DictModel[str, NonNegativeFloat]): - @root_validator(pre=True) - @classmethod - def ensure_negative_value_is_zero(cls, values): - # dasks adds/remove resource values and sometimes - # they end up being negative instead of 0 - if v := values.get("__root__", {}): - for res_key, res_value in v.items(): - if res_value < 0: - v[res_key] = 0 - return values - - -class Worker(BaseModel): - id: str - name: str - resources: AvailableResources - used_resources: UsedResources - memory_limit: ByteSize - metrics: WorkerMetrics - - -WorkersDict: TypeAlias = dict[AnyUrl, Worker] - - -class Scheduler(BaseModel): - status: str = Field(..., description="The running status of the scheduler") - workers: WorkersDict | None = Field(default_factory=dict) - - @validator("workers", pre=True, always=True) - @classmethod - def ensure_workers_is_empty_dict(cls, v): - if v is None: - return {} - return v - - -class ClusterDetails(BaseModel): - scheduler: Scheduler = Field( - ..., - description="This contains dask scheduler information given by the underlying dask library", - ) - dashboard_link: AnyUrl = Field( - ..., description="Link to this scheduler's dashboard" - ) - - -class ClusterGet(Cluster): - access_rights: dict[GroupID, ClusterAccessRights] = Field( - alias="accessRights", default_factory=dict - ) - - class Config(Cluster.Config): - allow_population_by_field_name = True - - @root_validator(pre=True) - @classmethod - def ensure_access_rights_converted(cls, values): - if "access_rights" in values: - access_rights = values.pop("access_rights") - values["accessRights"] = access_rights - return values - - -class ClusterDetailsGet(ClusterDetails): - ... - - -class ClusterCreate(BaseCluster): - owner: GroupID | None - authentication: ExternalClusterAuthentication - access_rights: dict[GroupID, ClusterAccessRights] = Field( - alias="accessRights", default_factory=dict - ) - - @validator("thumbnail", always=True, pre=True) - @classmethod - def set_default_thumbnail_if_empty(cls, v, values): - if v is None: - cluster_type = values["type"] - default_thumbnails = { - ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", - ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", - } - return default_thumbnails[cluster_type] - return v - - class Config(BaseCluster.Config): - schema_extra = { - "examples": [ - { - "name": "My awesome cluster", - "type": ClusterTypeInModel.ON_PREMISE, - "endpoint": "https://registry.osparc-development.fake.dev", - "authentication": { - "type": "simple", - "username": "someuser", - "password": "somepassword", - }, - }, - { - "name": "My AWS cluster", - "description": "a AWS cluster administered by me", - "type": ClusterTypeInModel.AWS, - "owner": 154, - "endpoint": "https://registry.osparc-development.fake.dev", - "authentication": { - "type": "simple", - "username": "someuser", - "password": "somepassword", - }, - "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, - }, - }, - ] - } - - -class ClusterPatch(BaseCluster): - name: str | None - description: str | None - type: ClusterTypeInModel | None - owner: GroupID | None - thumbnail: HttpUrl | None - endpoint: AnyUrl | None - authentication: ExternalClusterAuthentication | None - access_rights: dict[GroupID, ClusterAccessRights] | None = Field( - alias="accessRights" - ) - - class Config(BaseCluster.Config): - schema_extra = { - "examples": [ - { - "name": "Changing the name of my cluster", - }, - { - "description": "adding a better description", - }, - { - "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, - }, - }, - ] - } - - -class ClusterPing(BaseModel): - endpoint: AnyHttpUrl - authentication: ClusterAuthentication = Field( - ..., description="Dask gateway authentication" - ) diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/constants.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/constants.py deleted file mode 100644 index 8a2b7e8e94c..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Final - -# dynamic services - -DYNAMIC_SIDECAR_SERVICE_PREFIX: Final[str] = "dy-sidecar" -DYNAMIC_PROXY_SERVICE_PREFIX: Final[str] = "dy-proxy" -DYNAMIC_VOLUME_REMOVER_PREFIX: Final[str] = "dy-volrm" - -# label storing scheduler_data to allow service -# monitoring recovery after director-v2 reboots -DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: Final[str] = "io.simcore.scheduler-data" - -# This matches registries by: -# - local -# - itisfoundation -# - 10.0.0.0:8473 (IP & Port) -DYNAMIC_SIDECAR_DOCKER_IMAGE_RE = ( - r"(^([_a-zA-Z0-9:.-]+)/)?(dynamic-sidecar):([_a-zA-Z0-9.-]+$)" -) - -REGEX_DY_SERVICE_SIDECAR = rf"^{DYNAMIC_SIDECAR_SERVICE_PREFIX}_[a-zA-Z0-9-_]*" -REGEX_DY_SERVICE_PROXY = rf"^{DYNAMIC_PROXY_SERVICE_PREFIX}_[a-zA-Z0-9-_]*" diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/__init__.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/__init__.py deleted file mode 100644 index db7fb1894e1..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .scheduler import * -from .service import * diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py deleted file mode 100644 index ab54982b893..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py +++ /dev/null @@ -1,476 +0,0 @@ -import json -import logging -import re -from enum import Enum -from functools import cached_property -from typing import Any, Mapping, TypeAlias -from uuid import UUID - -from models_library.basic_types import PortInt -from models_library.generated_models.docker_rest_api import ContainerState, Status2 -from models_library.projects_nodes_io import NodeID -from models_library.service_settings_labels import ( - DynamicSidecarServiceLabels, - PathMappingsLabel, - SimcoreServiceLabels, -) -from models_library.services import RunID -from models_library.services_resources import ServiceResourcesDict -from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as -from servicelib.error_codes import ErrorCodeStr -from servicelib.exception_utils import DelayedExceptionHandler - -from ..constants import ( - DYNAMIC_PROXY_SERVICE_PREFIX, - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, - REGEX_DY_SERVICE_PROXY, - REGEX_DY_SERVICE_SIDECAR, -) -from .service import CommonServiceDetails - -TEMPORARY_PORT_NUMBER = 65_534 - -MAX_ALLOWED_SERVICE_NAME_LENGTH: int = 63 - - -DockerStatus: TypeAlias = Status2 - - -class DockerId(ConstrainedStr): - max_length = 25 - regex = re.compile(r"[A-Za-z0-9]{25}") - - -ServiceId: TypeAlias = DockerId -NetworkId: TypeAlias = DockerId - - -class ServiceName(ConstrainedStr): - strip_whitespace = True - min_length = 2 - - -logger = logging.getLogger() - - -def _strip_service_name(service_name: str) -> str: - """returns: the maximum allowed service name in docker swarm""" - return service_name[:MAX_ALLOWED_SERVICE_NAME_LENGTH] - - -def assemble_service_name(service_prefix: str, node_uuid: NodeID) -> str: - return _strip_service_name("_".join([service_prefix, str(node_uuid)])) - - -class DynamicSidecarStatus(str, Enum): - OK = "ok" # running as expected - FAILING = "failing" # requests to the sidecar API are failing service should be cosnidered as unavailable - - -class Status(BaseModel): - """Generated from data from docker container inspect API""" - - current: DynamicSidecarStatus = Field(..., description="status of the service") - info: str = Field(..., description="additional information for the user") - - def _update(self, new_status: DynamicSidecarStatus, new_info: str) -> None: - self.current = new_status - self.info = new_info - - def update_ok_status(self, info: str) -> None: - self._update(DynamicSidecarStatus.OK, info) - - def update_failing_status( - self, user_msg: str, error_code: ErrorCodeStr | None = None - ) -> None: - next_info = f"{user_msg}" - if error_code: - next_info = f"{user_msg} [{error_code}]" - - self._update(DynamicSidecarStatus.FAILING, next_info) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Status): - return NotImplemented - return self.current == other.current and self.info == other.info - - @classmethod - def create_as_initially_ok(cls) -> "Status": - # the service is initially ok when started - initial_state = cls(current=DynamicSidecarStatus.OK, info="") - return initial_state - - -class DockerContainerInspect(BaseModel): - container_state: ContainerState = Field( - ..., description="current state of container" - ) - name: str = Field(..., description="docker name of the container") - id: str = Field(..., description="docker id of the container") - - @cached_property - def status(self) -> DockerStatus: - assert self.container_state.Status # nosec - result: DockerStatus = self.container_state.Status - return result - - @classmethod - def from_container(cls, container: dict[str, Any]) -> "DockerContainerInspect": - return cls( - container_state=ContainerState(**container["State"]), - name=container["Name"], - id=container["Id"], - ) - - class Config: - keep_untouched = (cached_property,) - allow_mutation = False - - -class ServiceRemovalState(BaseModel): - can_remove: bool = Field( - False, - description="when True, marks the service as ready to be removed", - ) - can_save: bool = Field( - False, - description="when True, saves the internal state and upload outputs of the service", - ) - was_removed: bool = Field( - False, - description=( - "Will be True when the removal finished. Used primarily " - "to cancel retrying long running operations." - ), - ) - - def mark_to_remove(self, can_save: bool) -> None: - self.can_remove = True - self.can_save = can_save - - def mark_removed(self) -> None: - self.can_remove = False - self.was_removed = True - - -class DynamicSidecar(BaseModel): - status: Status = Field( - Status.create_as_initially_ok(), - description="status of the service sidecar also with additional information", - ) - - is_ready: bool = Field( - False, - scription=( - "is True while the health check on the dynamic-sidecar is responding. " - "Meaning that the dynamic-sidecar is reachable and can accept requests" - ), - ) - - @property - def compose_spec_submitted(self) -> bool: - """ - If the director-v2 is rebooted was_compose_spec_submitted is False - If the compose-spec is submitted it can be safely assumed that the - containers_inspect contains some elements. - """ - return self.was_compose_spec_submitted or len(self.containers_inspect) > 0 - - was_compose_spec_submitted: bool = Field( - False, - description="if the docker-compose spec was already submitted this fields is True", - ) - - containers_inspect: list[DockerContainerInspect] = Field( - [], - scription="docker inspect results from all the container ran at regular intervals", - ) - - was_dynamic_sidecar_started: bool = False - is_healthy: bool = False - were_containers_created: bool = Field( - False, - description=( - "when True no longer will the Docker api " - "be used to check if the services were started" - ), - ) - is_project_network_attached: bool = Field( - False, - description=( - "When True, all containers were in running state and project " - "networks were attached. Waiting for the container sto be in " - "running state guarantees all containers have been created" - ), - ) - - is_service_environment_ready: bool = Field( - False, - description=( - "True when the environment setup required by the " - "dynamic-sidecars created services was completed." - "Example: nodeports data downloaded, globally " - "shared service data fetched, etc.." - ), - ) - - service_removal_state: ServiceRemovalState = Field( - default_factory=ServiceRemovalState, - description=( - "stores information used during service removal " - "from the dynamic-sidecar scheduler" - ), - ) - - wait_for_manual_intervention_after_error: bool = Field( - False, - description=( - "Marks the sidecar as untouchable since there was an error and " - "important data might be lost. awaits for manual intervention." - ), - ) - wait_for_manual_intervention_logged: bool = Field( - False, description="True if a relative message was logged" - ) - were_state_and_outputs_saved: bool = Field( - False, - description="set True if the dy-sidecar saves the state and uploads the outputs", - ) - - # below had already been validated and - # used only to start the proxy - dynamic_sidecar_id: ServiceId | None = Field( - None, description="returned by the docker engine; used for starting the proxy" - ) - dynamic_sidecar_network_id: NetworkId | None = Field( - None, description="returned by the docker engine; used for starting the proxy" - ) - swarm_network_id: NetworkId | None = Field( - None, description="returned by the docker engine; used for starting the proxy" - ) - swarm_network_name: str | None = Field( - None, description="used for starting the proxy" - ) - - docker_node_id: str | None = Field( - None, - description=( - "contains node id of the docker node where all services " - "and created containers are started" - ), - ) - - inspect_error_handler: DelayedExceptionHandler = Field( - DelayedExceptionHandler(delay_for=0), - description=( - "Set when the dy-sidecar can no longer be reached by the " - "director-v2. If it will be possible to reach the dy-sidecar again, " - "this value will be set to None." - ), - ) - - class Config: - validate_assignment = True - - -class DynamicSidecarNamesHelper(BaseModel): - """ - Service naming schema: - NOTE: name is max 63 characters - dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf - - dynamic sidecar structure - 0. a network is created: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - 1. a dynamic-sidecar is started: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - a traefik instance: dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf - """ - - service_name_dynamic_sidecar: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, - description="unique name of the dynamic-sidecar service", - ) - proxy_service_name: str = Field( - ..., - regex=REGEX_DY_SERVICE_PROXY, - max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, - description="name of the proxy for the dynamic-sidecar", - ) - - simcore_traefik_zone: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - description="unique name for the traefik constraints", - ) - dynamic_sidecar_network_name: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - description="based on the node_id and project_id", - ) - - @classmethod - def make(cls, node_uuid: UUID) -> "DynamicSidecarNamesHelper": - return cls( - service_name_dynamic_sidecar=assemble_service_name( - DYNAMIC_SIDECAR_SERVICE_PREFIX, node_uuid - ), - proxy_service_name=assemble_service_name( - DYNAMIC_PROXY_SERVICE_PREFIX, node_uuid - ), - simcore_traefik_zone=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", - dynamic_sidecar_network_name=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", - ) - - -class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): - # TODO: ANE this object is just the context of the dy-sidecar. Should - # be called like so and subcontexts for different handlers should - # also be added. It will make keeping track of env vars more easily - - service_name: ServiceName = Field( - ..., - description="Name of the current dynamic-sidecar being observed", - ) - run_id: RunID = Field( - default_factory=RunID.create, - description=( - "Uniquely identify the dynamic sidecar session (a.k.a. 2 " - "subsequent exact same services will have a different run_id)" - ), - ) - hostname: str = Field( - ..., description="dy-sidecar's service hostname (provided by docker-swarm)" - ) - port: PortInt = Field(8000, description="dynamic-sidecar port") - - @property - def endpoint(self) -> AnyHttpUrl: - """endpoint where all the services are exposed""" - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, f"http://{self.hostname}:{self.port}" # NOSONAR - ) - return url - - dynamic_sidecar: DynamicSidecar = Field( - ..., - description="stores information fetched from the dynamic-sidecar", - ) - - paths_mapping: PathMappingsLabel # overwrites in DynamicSidecarServiceLabels - - dynamic_sidecar_network_name: str = Field( - ..., - description="overlay network biding the proxy to the container spaned by the dynamic-sidecar", - ) - - simcore_traefik_zone: str = Field( - ..., - description="required for Traefik to correctly route requests to the spawned container", - ) - - service_port: PortInt = Field( - TEMPORARY_PORT_NUMBER, - description=( - "port where the service is exposed defined by the service; " - "NOTE: temporary default because it will be changed once the service " - "is started, this value is fetched from the service start spec" - ), - ) - - service_resources: ServiceResourcesDict = Field( - ..., description="service resources used to enforce limits" - ) - - request_dns: str = Field( - ..., description="used when configuring the CORS options on the proxy" - ) - request_scheme: str = Field( - ..., description="used when configuring the CORS options on the proxy" - ) - request_simcore_user_agent: str = Field( - ..., - description="used as label to filter out the metrics from the cAdvisor prometheus metrics", - ) - proxy_service_name: str = Field(None, description="service name given to the proxy") - proxy_admin_api_port: PortInt | None = Field( - default=None, description="used as the admin endpoint API port" - ) - - @property - def get_proxy_endpoint(self) -> AnyHttpUrl: - """get the endpoint where the proxy's admin API is exposed""" - assert self.proxy_admin_api_port # nosec - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, f"http://{self.proxy_service_name}:{self.proxy_admin_api_port}" - ) - return url - - product_name: str = Field( - None, - description="Current product upon which this service is scheduled. " - "If set to None, the current product is undefined. Mostly for backwards compatibility", - ) - - @classmethod - def from_http_request( - # pylint: disable=too-many-arguments - cls, - service: "DynamicServiceCreate", # type: ignore - simcore_service_labels: SimcoreServiceLabels, - port: PortInt, - request_dns: str, - request_scheme: str, - request_simcore_user_agent: str, - can_save: bool, - run_id: RunID | None = None, - ) -> "SchedulerData": - # This constructor method sets current product - names_helper = DynamicSidecarNamesHelper.make(service.node_uuid) - - obj_dict = dict( - service_name=names_helper.service_name_dynamic_sidecar, - hostname=names_helper.service_name_dynamic_sidecar, - port=port, - node_uuid=service.node_uuid, - project_id=service.project_id, - user_id=service.user_id, - key=service.key, - version=service.version, - service_resources=service.service_resources, - product_name=service.product_name, - paths_mapping=simcore_service_labels.paths_mapping, - compose_spec=json.dumps(simcore_service_labels.compose_spec), - container_http_entry=simcore_service_labels.container_http_entry, - restart_policy=simcore_service_labels.restart_policy, - dynamic_sidecar_network_name=names_helper.dynamic_sidecar_network_name, - simcore_traefik_zone=names_helper.simcore_traefik_zone, - request_dns=request_dns, - request_scheme=request_scheme, - proxy_service_name=names_helper.proxy_service_name, - request_simcore_user_agent=request_simcore_user_agent, - dynamic_sidecar={"service_removal_state": {"can_save": can_save}}, - ) - if run_id: - obj_dict["run_id"] = run_id - return cls.parse_obj(obj_dict) - - @classmethod - def from_service_inspect( - cls, service_inspect: Mapping[str, Any] - ) -> "SchedulerData": - labels = service_inspect["Spec"]["Labels"] - return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) - - def as_label_data(self) -> str: - # compose_spec needs to be json encoded before encoding it to json - # and storing it in the label - return self.copy( - update={"compose_spec": json.dumps(self.compose_spec)}, deep=True - ).json() - - class Config: - extra = Extra.allow - allow_population_by_field_name = True diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/service.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/service.py deleted file mode 100644 index fbd32f78d88..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/service.py +++ /dev/null @@ -1,191 +0,0 @@ -from enum import Enum, unique -from functools import cached_property, lru_cache, total_ordering -from pathlib import Path -from typing import Any, ClassVar - -from models_library.basic_types import PortInt -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.services import VERSION_RE, DynamicServiceKey -from models_library.users import UserID -from pydantic import BaseModel, Field - - -class CommonServiceDetails(BaseModel): - key: DynamicServiceKey = Field( - ..., - description="distinctive name for the node based on the docker registry path", - examples=[ - "simcore/services/dynamic/3dviewer", - ], - alias="service_key", - ) - version: str = Field( - ..., - description="semantic version number of the node", - regex=VERSION_RE, - examples=["1.0.0", "0.0.1"], - alias="service_version", - ) - - user_id: UserID - project_id: ProjectID - node_uuid: NodeID = Field(..., alias="service_uuid") - - -class ServiceDetails(CommonServiceDetails): - basepath: Path = Field( - default=None, - description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.", - alias="service_basepath", - ) - - class Config: - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { - "example": { - "key": "simcore/services/dynamic/3dviewer", - "version": "2.4.5", - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", - } - } - - -@unique -class ServiceBootType(str, Enum): - V0 = "V0" - V2 = "V2" - - -@total_ordering -@unique -class ServiceState(Enum): - PENDING = "pending" - PULLING = "pulling" - STARTING = "starting" - RUNNING = "running" - COMPLETE = "complete" - FAILED = "failed" - STOPPING = "stopping" - - def __lt__(self, other): - if self.__class__ is other.__class__: - comparison_order = ServiceState.comparison_order() - self_index = comparison_order[self] - other_index = comparison_order[other] - return self_index < other_index - return NotImplemented - - @staticmethod - @lru_cache(maxsize=2) - def comparison_order() -> dict["ServiceState", int]: - """States are comparable to supportmin() on a list of ServiceState""" - return { - ServiceState.FAILED: 0, - ServiceState.PENDING: 1, - ServiceState.PULLING: 2, - ServiceState.STARTING: 3, - ServiceState.RUNNING: 4, - ServiceState.STOPPING: 5, - ServiceState.COMPLETE: 6, - } - - -class RunningDynamicServiceDetails(ServiceDetails): - boot_type: ServiceBootType = Field( - default=ServiceBootType.V0, - description=( - "Describes how the dynamic services was started (legacy=V0, modern=V2)." - "Since legacy services do not have this label it defaults to V0." - ), - ) - - host: str = Field( - ..., description="the service swarm internal host name", alias="service_host" - ) - internal_port: PortInt = Field( - ..., description="the service swarm internal port", alias="service_port" - ) - published_port: PortInt = Field( - default=None, - description="the service swarm published port if any", - deprecated=True, - ) - - entry_point: str | None = Field( - default=None, - description="if empty the service entrypoint is on the root endpoint.", - deprecated=True, - ) - state: ServiceState = Field( - ..., description="service current state", alias="service_state" - ) - message: str | None = Field( - default=None, - description="additional information related to service state", - alias="service_message", - ) - - @cached_property - def legacy_service_url(self) -> str: - return f"http://{self.host}:{self.internal_port}{self.basepath}" # NOSONAR - - @classmethod - def from_scheduler_data( - cls, - node_uuid: NodeID, - scheduler_data: "SchedulerData", # type: ignore - service_state: ServiceState, - service_message: str, - ) -> "RunningDynamicServiceDetails": - return cls.parse_obj( - { - "boot_type": ServiceBootType.V2, - "user_id": scheduler_data.user_id, - "project_id": scheduler_data.project_id, - "service_uuid": node_uuid, - "service_key": scheduler_data.key, - "service_version": scheduler_data.version, - "service_host": scheduler_data.service_name, - "service_port": scheduler_data.service_port, - "service_state": service_state.value, - "service_message": service_message, - } - ) - - class Config(ServiceDetails.Config): - keep_untouched = (cached_property,) - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [ - { - "boot_type": "V0", - "key": "simcore/services/dynamic/3dviewer", - "version": "2.4.5", - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "host": "3dviewer_75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "internal_port": 8888, - "state": "running", - "message": "", - "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - }, - { - "boot_type": "V2", - "key": "simcore/services/dynamic/dy-static-file-viewer-dynamic-sidecar", - "version": "1.0.0", - "user_id": 234, - "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", - "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "host": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa", - "internal_port": 80, - "state": "running", - "message": "", - "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", - }, - ] - } diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/meta.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/meta.py deleted file mode 100644 index 037fb73df15..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/meta.py +++ /dev/null @@ -1,25 +0,0 @@ -import re - -from models_library.basic_regex import VERSION_RE -from pydantic import BaseModel, ConstrainedStr, Field - - -class VersionStr(ConstrainedStr): - regex = re.compile(VERSION_RE) - - -class Meta(BaseModel): - name: str - version: VersionStr - released: dict[str, VersionStr] | None = Field( - None, description="Maps every route's path tag with a released version" - ) - - class Config: - schema_extra = { - "example": { - "name": "simcore_service_foo", - "version": "2.4.45", - "released": {"v1": "1.3.4", "v2": "2.4.45"}, - } - } From b3e8da1922133073f06db4cfb21b50a948b52c04 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:52:12 +0200 Subject: [PATCH 06/20] minor meta --- .../src/models_library/api_schemas_directorv2/meta.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/meta.py b/packages/models-library/src/models_library/api_schemas_directorv2/meta.py index ef140792d38..edc2f904000 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/meta.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/meta.py @@ -1,13 +1,8 @@ -import re from typing import Any, ClassVar -from pydantic import BaseModel, ConstrainedStr, Field +from pydantic import BaseModel, Field -from ..basic_regex import VERSION_RE - - -class VersionStr(ConstrainedStr): - regex = re.compile(VERSION_RE) +from ..basic_types import VersionStr class Meta(BaseModel): From 6adef9a386c5075d07c3e7a37d403a76bbeb56b8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:52:33 +0200 Subject: [PATCH 07/20] dynamic_services_scheduler is a domain model --- .../dynamic_services_scheduler.py | 485 ------------------ 1 file changed, 485 deletions(-) delete mode 100644 packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py deleted file mode 100644 index 9778a3b7c4f..00000000000 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_scheduler.py +++ /dev/null @@ -1,485 +0,0 @@ -import json -import logging -import re -from collections.abc import Mapping -from enum import Enum -from functools import cached_property -from typing import Any, TypeAlias -from uuid import UUID - -from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as -from servicelib.error_codes import ErrorCodeStr -from servicelib.exception_utils import DelayedExceptionHandler - -from ..api_schemas_directorv2.dynamic_services_service import CommonServiceDetails -from ..basic_types import PortInt -from ..generated_models.docker_rest_api import ContainerState, Status2 -from ..projects_nodes_io import NodeID -from ..service_settings_labels import ( - DynamicSidecarServiceLabels, - PathMappingsLabel, - SimcoreServiceLabels, -) -from ..services import RunID -from ..services_resources import ServiceResourcesDict -from .constants import ( - DYNAMIC_PROXY_SERVICE_PREFIX, - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, - REGEX_DY_SERVICE_PROXY, - REGEX_DY_SERVICE_SIDECAR, -) - -TEMPORARY_PORT_NUMBER = 65_534 - -MAX_ALLOWED_SERVICE_NAME_LENGTH: int = 63 - - -DockerStatus: TypeAlias = Status2 - - -class DockerId(ConstrainedStr): - max_length = 25 - regex = re.compile(r"[A-Za-z0-9]{25}") - - -ServiceId: TypeAlias = DockerId -NetworkId: TypeAlias = DockerId - - -class ServiceName(ConstrainedStr): - strip_whitespace = True - min_length = 2 - - -logger = logging.getLogger() - - -def _strip_service_name(service_name: str) -> str: - """returns: the maximum allowed service name in docker swarm""" - return service_name[:MAX_ALLOWED_SERVICE_NAME_LENGTH] - - -def assemble_service_name(service_prefix: str, node_uuid: NodeID) -> str: - return _strip_service_name("_".join([service_prefix, str(node_uuid)])) - - -class DynamicSidecarStatus(str, Enum): - OK = "ok" # running as expected - FAILING = "failing" # requests to the sidecar API are failing service should be cosnidered as unavailable - - -class Status(BaseModel): - """Generated from data from docker container inspect API""" - - current: DynamicSidecarStatus = Field(..., description="status of the service") - info: str = Field(..., description="additional information for the user") - - def _update(self, new_status: DynamicSidecarStatus, new_info: str) -> None: - self.current = new_status - self.info = new_info - - def update_ok_status(self, info: str) -> None: - self._update(DynamicSidecarStatus.OK, info) - - def update_failing_status( - self, user_msg: str, error_code: ErrorCodeStr | None = None - ) -> None: - next_info = f"{user_msg}" - if error_code: - next_info = f"{user_msg} [{error_code}]" - - self._update(DynamicSidecarStatus.FAILING, next_info) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Status): - return NotImplemented - return self.current == other.current and self.info == other.info - - @classmethod - def create_as_initially_ok(cls) -> "Status": - # the service is initially ok when started - initial_state: "Status" = cls(current=DynamicSidecarStatus.OK, info="") - return initial_state - - -class DockerContainerInspect(BaseModel): - container_state: ContainerState = Field( - ..., description="current state of container" - ) - name: str = Field(..., description="docker name of the container") - id: str = Field(..., description="docker id of the container") - - @cached_property - def status(self) -> DockerStatus: - assert self.container_state.Status # nosec - result: DockerStatus = self.container_state.Status - return result - - @classmethod - def from_container(cls, container: dict[str, Any]) -> "DockerContainerInspect": - return cls( - container_state=ContainerState(**container["State"]), - name=container["Name"], - id=container["Id"], - ) - - class Config: - keep_untouched = (cached_property,) - allow_mutation = False - - -class ServiceRemovalState(BaseModel): - can_remove: bool = Field( - default=False, - description="when True, marks the service as ready to be removed", - ) - can_save: bool = Field( - default=False, - description="when True, saves the internal state and upload outputs of the service", - ) - was_removed: bool = Field( - default=False, - description=( - "Will be True when the removal finished. Used primarily " - "to cancel retrying long running operations." - ), - ) - - def mark_to_remove(self, *, can_save: bool) -> None: - self.can_remove = True - self.can_save = can_save - - def mark_removed(self) -> None: - self.can_remove = False - self.was_removed = True - - -class DynamicSidecar(BaseModel): - status: Status = Field( - Status.create_as_initially_ok(), - description="status of the service sidecar also with additional information", - ) - - is_ready: bool = Field( - default=False, - scription=( - "is True while the health check on the dynamic-sidecar is responding. " - "Meaning that the dynamic-sidecar is reachable and can accept requests" - ), - ) - - @property - def compose_spec_submitted(self) -> bool: - """ - If the director-v2 is rebooted was_compose_spec_submitted is False - If the compose-spec is submitted it can be safely assumed that the - containers_inspect contains some elements. - """ - return self.was_compose_spec_submitted or len(self.containers_inspect) > 0 - - was_compose_spec_submitted: bool = Field( - default=False, - description="if the docker-compose spec was already submitted this fields is True", - ) - - containers_inspect: list[DockerContainerInspect] = Field( - [], - scription="docker inspect results from all the container ran at regular intervals", - ) - - was_dynamic_sidecar_started: bool = False - is_healthy: bool = False - were_containers_created: bool = Field( - default=False, - description=( - "when True no longer will the Docker api " - "be used to check if the services were started" - ), - ) - is_project_network_attached: bool = Field( - default=False, - description=( - "When True, all containers were in running state and project " - "networks were attached. Waiting for the container sto be in " - "running state guarantees all containers have been created" - ), - ) - - is_service_environment_ready: bool = Field( - default=False, - description=( - "True when the environment setup required by the " - "dynamic-sidecars created services was completed." - "Example: nodeports data downloaded, globally " - "shared service data fetched, etc.." - ), - ) - - service_removal_state: ServiceRemovalState = Field( - default_factory=ServiceRemovalState, - description=( - "stores information used during service removal " - "from the dynamic-sidecar scheduler" - ), - ) - - wait_for_manual_intervention_after_error: bool = Field( - default=False, - description=( - "Marks the sidecar as untouchable since there was an error and " - "important data might be lost. awaits for manual intervention." - ), - ) - wait_for_manual_intervention_logged: bool = Field( - default=False, description="True if a relative message was logged" - ) - were_state_and_outputs_saved: bool = Field( - default=False, - description="set True if the dy-sidecar saves the state and uploads the outputs", - ) - - # below had already been validated and - # used only to start the proxy - dynamic_sidecar_id: ServiceId | None = Field( - default=None, - description="returned by the docker engine; used for starting the proxy", - ) - dynamic_sidecar_network_id: NetworkId | None = Field( - default=None, - description="returned by the docker engine; used for starting the proxy", - ) - swarm_network_id: NetworkId | None = Field( - default=None, - description="returned by the docker engine; used for starting the proxy", - ) - swarm_network_name: str | None = Field( - default=None, description="used for starting the proxy" - ) - - docker_node_id: str | None = Field( - default=None, - description=( - "contains node id of the docker node where all services " - "and created containers are started" - ), - ) - - inspect_error_handler: DelayedExceptionHandler = Field( - default=DelayedExceptionHandler(delay_for=0), - description=( - "Set when the dy-sidecar can no longer be reached by the " - "director-v2. If it will be possible to reach the dy-sidecar again, " - "this value will be set to None." - ), - ) - - class Config: - validate_assignment = True - - -class DynamicSidecarNamesHelper(BaseModel): - """ - Service naming schema: - NOTE: name is max 63 characters - dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf - - dynamic sidecar structure - 0. a network is created: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - 1. a dynamic-sidecar is started: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf - a traefik instance: dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf - """ - - service_name_dynamic_sidecar: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, - description="unique name of the dynamic-sidecar service", - ) - proxy_service_name: str = Field( - ..., - regex=REGEX_DY_SERVICE_PROXY, - max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, - description="name of the proxy for the dynamic-sidecar", - ) - - simcore_traefik_zone: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - description="unique name for the traefik constraints", - ) - dynamic_sidecar_network_name: str = Field( - ..., - regex=REGEX_DY_SERVICE_SIDECAR, - description="based on the node_id and project_id", - ) - - @classmethod - def make(cls, node_uuid: UUID) -> "DynamicSidecarNamesHelper": - return cls( - service_name_dynamic_sidecar=assemble_service_name( - DYNAMIC_SIDECAR_SERVICE_PREFIX, node_uuid - ), - proxy_service_name=assemble_service_name( - DYNAMIC_PROXY_SERVICE_PREFIX, node_uuid - ), - simcore_traefik_zone=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", - dynamic_sidecar_network_name=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", - ) - - -class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): - # TODO: ANE this object is just the context of the dy-sidecar. Should - # be called like so and subcontexts for different handlers should - # also be added. It will make keeping track of env vars more easily - - service_name: ServiceName = Field( - ..., - description="Name of the current dynamic-sidecar being observed", - ) - run_id: RunID = Field( - default_factory=RunID.create, - description=( - "Uniquely identify the dynamic sidecar session (a.k.a. 2 " - "subsequent exact same services will have a different run_id)" - ), - ) - hostname: str = Field( - ..., description="dy-sidecar's service hostname (provided by docker-swarm)" - ) - port: PortInt = Field( - default=parse_obj_as(PortInt, 8000), description="dynamic-sidecar port" - ) - - @property - def endpoint(self) -> AnyHttpUrl: - """endpoint where all the services are exposed""" - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, f"http://{self.hostname}:{self.port}" # NOSONAR - ) - return url - - dynamic_sidecar: DynamicSidecar = Field( - ..., - description="stores information fetched from the dynamic-sidecar", - ) - - paths_mapping: PathMappingsLabel # overwrites in DynamicSidecarServiceLabels - - dynamic_sidecar_network_name: str = Field( - ..., - description="overlay network biding the proxy to the container spaned by the dynamic-sidecar", - ) - - simcore_traefik_zone: str = Field( - ..., - description="required for Traefik to correctly route requests to the spawned container", - ) - - service_port: PortInt = Field( - default=parse_obj_as(PortInt, TEMPORARY_PORT_NUMBER), - description=( - "port where the service is exposed defined by the service; " - "NOTE: temporary default because it will be changed once the service " - "is started, this value is fetched from the service start spec" - ), - ) - - service_resources: ServiceResourcesDict = Field( - ..., description="service resources used to enforce limits" - ) - - request_dns: str = Field( - ..., description="used when configuring the CORS options on the proxy" - ) - request_scheme: str = Field( - ..., description="used when configuring the CORS options on the proxy" - ) - request_simcore_user_agent: str = Field( - ..., - description="used as label to filter out the metrics from the cAdvisor prometheus metrics", - ) - proxy_service_name: str = Field( - default=None, description="service name given to the proxy" - ) - proxy_admin_api_port: PortInt | None = Field( - default=None, description="used as the admin endpoint API port" - ) - - @property - def get_proxy_endpoint(self) -> AnyHttpUrl: - """get the endpoint where the proxy's admin API is exposed""" - assert self.proxy_admin_api_port # nosec - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, - f"http://{self.proxy_service_name}:{self.proxy_admin_api_port}", # nosec # NOSONAR - ) - return url - - product_name: str = Field( - None, - description="Current product upon which this service is scheduled. " - "If set to None, the current product is undefined. Mostly for backwards compatibility", - ) - - @classmethod - def from_http_request( - # pylint: disable=too-many-arguments - cls, - service: "DynamicServiceCreate", # type: ignore - simcore_service_labels: SimcoreServiceLabels, - port: PortInt, - request_dns: str, - request_scheme: str, - request_simcore_user_agent: str, - can_save: bool, - run_id: RunID | None = None, - ) -> "SchedulerData": - # This constructor method sets current product - names_helper = DynamicSidecarNamesHelper.make(service.node_uuid) - - obj_dict = { - "service_name": names_helper.service_name_dynamic_sidecar, - "hostname": names_helper.service_name_dynamic_sidecar, - "port": port, - "node_uuid": service.node_uuid, - "project_id": service.project_id, - "user_id": service.user_id, - "key": service.key, - "version": service.version, - "service_resources": service.service_resources, - "product_name": service.product_name, - "paths_mapping": simcore_service_labels.paths_mapping, - "compose_spec": json.dumps(simcore_service_labels.compose_spec), - "container_http_entry": simcore_service_labels.container_http_entry, - "restart_policy": simcore_service_labels.restart_policy, - "dynamic_sidecar_network_name": names_helper.dynamic_sidecar_network_name, - "simcore_traefik_zone": names_helper.simcore_traefik_zone, - "request_dns": request_dns, - "request_scheme": request_scheme, - "proxy_service_name": names_helper.proxy_service_name, - "request_simcore_user_agent": request_simcore_user_agent, - "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}}, - } - if run_id: - obj_dict["run_id"] = run_id - return cls.parse_obj(obj_dict) - - @classmethod - def from_service_inspect( - cls, service_inspect: Mapping[str, Any] - ) -> "SchedulerData": - labels = service_inspect["Spec"]["Labels"] - return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) - - def as_label_data(self) -> str: - # compose_spec needs to be json encoded before encoding it to json - # and storing it in the label - return self.copy( - update={"compose_spec": json.dumps(self.compose_spec)}, deep=True - ).json() - - class Config: - extra = Extra.allow - allow_population_by_field_name = True From d34762cb6f4a52116d63018026b6c41fdf70b368 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:53:05 +0200 Subject: [PATCH 08/20] dynamic service scheduler is a domain model --- .../simcore_service_director_v2/cli/_core.py | 6 +- .../models/dynamic_services_scheduler.py | 486 ++++++++++++++++++ .../dynamic_sidecar/api_client/_public.py | 4 +- .../dynamic_sidecar/docker_api/_core.py | 10 +- .../docker_service_specs/proxy.py | 4 +- .../docker_service_specs/sidecar.py | 4 +- .../modules/dynamic_sidecar/docker_states.py | 6 +- .../dynamic_sidecar/scheduler/_core/_abc.py | 4 +- .../scheduler/_core/_events.py | 14 +- .../scheduler/_core/_events_utils.py | 10 +- .../scheduler/_core/_observer.py | 4 +- .../scheduler/_core/_scheduler.py | 8 +- .../scheduler/_core/_scheduler_utils.py | 8 +- .../director-v2/tests/integration/02/utils.py | 4 +- .../unit/test_api_route_dynamic_scheduler.py | 4 +- .../unit/test_models_dynamic_services.py | 2 +- ...ic_sidecar_docker_service_specs_sidecar.py | 4 +- .../test_modules_dynamic_sidecar_scheduler.py | 6 +- ...s_dynamic_sidecar_scheduler_core_events.py | 10 +- ..._modules_dynamic_sidecar_scheduler_task.py | 4 +- ...test_schemas_dynamic_services_scheduler.py | 4 +- .../test_api_route_dynamic_services.py | 4 +- ...test_modules_dynamic_sidecar_docker_api.py | 8 +- 23 files changed, 542 insertions(+), 76 deletions(-) create mode 100644 services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_core.py b/services/director-v2/src/simcore_service_director_v2/cli/_core.py index c27277048dc..81b910c370c 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_core.py @@ -8,9 +8,6 @@ from fastapi import FastAPI, status from httpx import AsyncClient, HTTPError from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DynamicSidecarNamesHelper, -) from models_library.projects import NodeIDStr, ProjectID from models_library.projects_nodes_io import NodeID from models_library.services import ServiceType @@ -19,6 +16,9 @@ from rich.live import Live from rich.table import Table from servicelib.services_utils import get_service_from_key +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DynamicSidecarNamesHelper, +) from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_attempt from tenacity.wait import wait_random_exponential diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py new file mode 100644 index 00000000000..944bc6c0d8c --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -0,0 +1,486 @@ +import json +import logging +import re +from collections.abc import Mapping +from enum import Enum +from functools import cached_property +from typing import Any, TypeAlias +from uuid import UUID + +from models_library.api_schemas_directorv2.constants import ( + DYNAMIC_PROXY_SERVICE_PREFIX, + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, + REGEX_DY_SERVICE_PROXY, + REGEX_DY_SERVICE_SIDECAR, +) +from models_library.api_schemas_directorv2.dynamic_services_service import ( + CommonServiceDetails, +) +from models_library.basic_types import PortInt +from models_library.generated_models.docker_rest_api import ContainerState, Status2 +from models_library.projects_nodes_io import NodeID +from models_library.service_settings_labels import ( + DynamicSidecarServiceLabels, + PathMappingsLabel, + SimcoreServiceLabels, +) +from models_library.services import RunID +from models_library.services_resources import ServiceResourcesDict +from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as +from servicelib.error_codes import ErrorCodeStr +from servicelib.exception_utils import DelayedExceptionHandler + +TEMPORARY_PORT_NUMBER = 65_534 + +MAX_ALLOWED_SERVICE_NAME_LENGTH: int = 63 + + +DockerStatus: TypeAlias = Status2 + + +class DockerId(ConstrainedStr): + max_length = 25 + regex = re.compile(r"[A-Za-z0-9]{25}") + + +ServiceId: TypeAlias = DockerId +NetworkId: TypeAlias = DockerId + + +class ServiceName(ConstrainedStr): + strip_whitespace = True + min_length = 2 + + +logger = logging.getLogger() + + +def _strip_service_name(service_name: str) -> str: + """returns: the maximum allowed service name in docker swarm""" + return service_name[:MAX_ALLOWED_SERVICE_NAME_LENGTH] + + +def assemble_service_name(service_prefix: str, node_uuid: NodeID) -> str: + return _strip_service_name("_".join([service_prefix, str(node_uuid)])) + + +class DynamicSidecarStatus(str, Enum): + OK = "ok" # running as expected + FAILING = "failing" # requests to the sidecar API are failing service should be cosnidered as unavailable + + +class Status(BaseModel): + """Generated from data from docker container inspect API""" + + current: DynamicSidecarStatus = Field(..., description="status of the service") + info: str = Field(..., description="additional information for the user") + + def _update(self, new_status: DynamicSidecarStatus, new_info: str) -> None: + self.current = new_status + self.info = new_info + + def update_ok_status(self, info: str) -> None: + self._update(DynamicSidecarStatus.OK, info) + + def update_failing_status( + self, user_msg: str, error_code: ErrorCodeStr | None = None + ) -> None: + next_info = f"{user_msg}" + if error_code: + next_info = f"{user_msg} [{error_code}]" + + self._update(DynamicSidecarStatus.FAILING, next_info) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Status): + return NotImplemented + return self.current == other.current and self.info == other.info + + @classmethod + def create_as_initially_ok(cls) -> "Status": + # the service is initially ok when started + initial_state: "Status" = cls(current=DynamicSidecarStatus.OK, info="") + return initial_state + + +class DockerContainerInspect(BaseModel): + container_state: ContainerState = Field( + ..., description="current state of container" + ) + name: str = Field(..., description="docker name of the container") + id: str = Field(..., description="docker id of the container") + + @cached_property + def status(self) -> DockerStatus: + assert self.container_state.Status # nosec + result: DockerStatus = self.container_state.Status + return result + + @classmethod + def from_container(cls, container: dict[str, Any]) -> "DockerContainerInspect": + return cls( + container_state=ContainerState(**container["State"]), + name=container["Name"], + id=container["Id"], + ) + + class Config: + keep_untouched = (cached_property,) + allow_mutation = False + + +class ServiceRemovalState(BaseModel): + can_remove: bool = Field( + default=False, + description="when True, marks the service as ready to be removed", + ) + can_save: bool = Field( + default=False, + description="when True, saves the internal state and upload outputs of the service", + ) + was_removed: bool = Field( + default=False, + description=( + "Will be True when the removal finished. Used primarily " + "to cancel retrying long running operations." + ), + ) + + def mark_to_remove(self, *, can_save: bool) -> None: + self.can_remove = True + self.can_save = can_save + + def mark_removed(self) -> None: + self.can_remove = False + self.was_removed = True + + +class DynamicSidecar(BaseModel): + status: Status = Field( + Status.create_as_initially_ok(), + description="status of the service sidecar also with additional information", + ) + + is_ready: bool = Field( + default=False, + scription=( + "is True while the health check on the dynamic-sidecar is responding. " + "Meaning that the dynamic-sidecar is reachable and can accept requests" + ), + ) + + @property + def compose_spec_submitted(self) -> bool: + """ + If the director-v2 is rebooted was_compose_spec_submitted is False + If the compose-spec is submitted it can be safely assumed that the + containers_inspect contains some elements. + """ + return self.was_compose_spec_submitted or len(self.containers_inspect) > 0 + + was_compose_spec_submitted: bool = Field( + default=False, + description="if the docker-compose spec was already submitted this fields is True", + ) + + containers_inspect: list[DockerContainerInspect] = Field( + [], + scription="docker inspect results from all the container ran at regular intervals", + ) + + was_dynamic_sidecar_started: bool = False + is_healthy: bool = False + were_containers_created: bool = Field( + default=False, + description=( + "when True no longer will the Docker api " + "be used to check if the services were started" + ), + ) + is_project_network_attached: bool = Field( + default=False, + description=( + "When True, all containers were in running state and project " + "networks were attached. Waiting for the container sto be in " + "running state guarantees all containers have been created" + ), + ) + + is_service_environment_ready: bool = Field( + default=False, + description=( + "True when the environment setup required by the " + "dynamic-sidecars created services was completed." + "Example: nodeports data downloaded, globally " + "shared service data fetched, etc.." + ), + ) + + service_removal_state: ServiceRemovalState = Field( + default_factory=ServiceRemovalState, + description=( + "stores information used during service removal " + "from the dynamic-sidecar scheduler" + ), + ) + + wait_for_manual_intervention_after_error: bool = Field( + default=False, + description=( + "Marks the sidecar as untouchable since there was an error and " + "important data might be lost. awaits for manual intervention." + ), + ) + wait_for_manual_intervention_logged: bool = Field( + default=False, description="True if a relative message was logged" + ) + were_state_and_outputs_saved: bool = Field( + default=False, + description="set True if the dy-sidecar saves the state and uploads the outputs", + ) + + # below had already been validated and + # used only to start the proxy + dynamic_sidecar_id: ServiceId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + dynamic_sidecar_network_id: NetworkId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + swarm_network_id: NetworkId | None = Field( + default=None, + description="returned by the docker engine; used for starting the proxy", + ) + swarm_network_name: str | None = Field( + default=None, description="used for starting the proxy" + ) + + docker_node_id: str | None = Field( + default=None, + description=( + "contains node id of the docker node where all services " + "and created containers are started" + ), + ) + + inspect_error_handler: DelayedExceptionHandler = Field( + default=DelayedExceptionHandler(delay_for=0), + description=( + "Set when the dy-sidecar can no longer be reached by the " + "director-v2. If it will be possible to reach the dy-sidecar again, " + "this value will be set to None." + ), + ) + + class Config: + validate_assignment = True + + +class DynamicSidecarNamesHelper(BaseModel): + """ + Service naming schema: + NOTE: name is max 63 characters + dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf + + dynamic sidecar structure + 0. a network is created: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + 1. a dynamic-sidecar is started: dy-sidecar_4dde07ea-73be-4c44-845a-89479d1556cf + a traefik instance: dy-proxy_4dde07ea-73be-4c44-845a-89479d1556cf + """ + + service_name_dynamic_sidecar: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, + description="unique name of the dynamic-sidecar service", + ) + proxy_service_name: str = Field( + ..., + regex=REGEX_DY_SERVICE_PROXY, + max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, + description="name of the proxy for the dynamic-sidecar", + ) + + simcore_traefik_zone: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + description="unique name for the traefik constraints", + ) + dynamic_sidecar_network_name: str = Field( + ..., + regex=REGEX_DY_SERVICE_SIDECAR, + description="based on the node_id and project_id", + ) + + @classmethod + def make(cls, node_uuid: UUID) -> "DynamicSidecarNamesHelper": + return cls( + service_name_dynamic_sidecar=assemble_service_name( + DYNAMIC_SIDECAR_SERVICE_PREFIX, node_uuid + ), + proxy_service_name=assemble_service_name( + DYNAMIC_PROXY_SERVICE_PREFIX, node_uuid + ), + simcore_traefik_zone=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", + dynamic_sidecar_network_name=f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}", + ) + + +class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): + # TODO: ANE this object is just the context of the dy-sidecar. Should + # be called like so and subcontexts for different handlers should + # also be added. It will make keeping track of env vars more easily + + service_name: ServiceName = Field( + ..., + description="Name of the current dynamic-sidecar being observed", + ) + run_id: RunID = Field( + default_factory=RunID.create, + description=( + "Uniquely identify the dynamic sidecar session (a.k.a. 2 " + "subsequent exact same services will have a different run_id)" + ), + ) + hostname: str = Field( + ..., description="dy-sidecar's service hostname (provided by docker-swarm)" + ) + port: PortInt = Field( + default=parse_obj_as(PortInt, 8000), description="dynamic-sidecar port" + ) + + @property + def endpoint(self) -> AnyHttpUrl: + """endpoint where all the services are exposed""" + url: AnyHttpUrl = parse_obj_as( + AnyHttpUrl, f"http://{self.hostname}:{self.port}" # NOSONAR + ) + return url + + dynamic_sidecar: DynamicSidecar = Field( + ..., + description="stores information fetched from the dynamic-sidecar", + ) + + paths_mapping: PathMappingsLabel # overwrites in DynamicSidecarServiceLabels + + dynamic_sidecar_network_name: str = Field( + ..., + description="overlay network biding the proxy to the container spaned by the dynamic-sidecar", + ) + + simcore_traefik_zone: str = Field( + ..., + description="required for Traefik to correctly route requests to the spawned container", + ) + + service_port: PortInt = Field( + default=parse_obj_as(PortInt, TEMPORARY_PORT_NUMBER), + description=( + "port where the service is exposed defined by the service; " + "NOTE: temporary default because it will be changed once the service " + "is started, this value is fetched from the service start spec" + ), + ) + + service_resources: ServiceResourcesDict = Field( + ..., description="service resources used to enforce limits" + ) + + request_dns: str = Field( + ..., description="used when configuring the CORS options on the proxy" + ) + request_scheme: str = Field( + ..., description="used when configuring the CORS options on the proxy" + ) + request_simcore_user_agent: str = Field( + ..., + description="used as label to filter out the metrics from the cAdvisor prometheus metrics", + ) + proxy_service_name: str = Field( + default=None, description="service name given to the proxy" + ) + proxy_admin_api_port: PortInt | None = Field( + default=None, description="used as the admin endpoint API port" + ) + + @property + def get_proxy_endpoint(self) -> AnyHttpUrl: + """get the endpoint where the proxy's admin API is exposed""" + assert self.proxy_admin_api_port # nosec + url: AnyHttpUrl = parse_obj_as( + AnyHttpUrl, + f"http://{self.proxy_service_name}:{self.proxy_admin_api_port}", # nosec # NOSONAR + ) + return url + + product_name: str = Field( + None, + description="Current product upon which this service is scheduled. " + "If set to None, the current product is undefined. Mostly for backwards compatibility", + ) + + @classmethod + def from_http_request( + # pylint: disable=too-many-arguments + cls, + service: "DynamicServiceCreate", # type: ignore + simcore_service_labels: SimcoreServiceLabels, + port: PortInt, + request_dns: str, + request_scheme: str, + request_simcore_user_agent: str, + can_save: bool, + run_id: RunID | None = None, + ) -> "SchedulerData": + # This constructor method sets current product + names_helper = DynamicSidecarNamesHelper.make(service.node_uuid) + + obj_dict = { + "service_name": names_helper.service_name_dynamic_sidecar, + "hostname": names_helper.service_name_dynamic_sidecar, + "port": port, + "node_uuid": service.node_uuid, + "project_id": service.project_id, + "user_id": service.user_id, + "key": service.key, + "version": service.version, + "service_resources": service.service_resources, + "product_name": service.product_name, + "paths_mapping": simcore_service_labels.paths_mapping, + "compose_spec": json.dumps(simcore_service_labels.compose_spec), + "container_http_entry": simcore_service_labels.container_http_entry, + "restart_policy": simcore_service_labels.restart_policy, + "dynamic_sidecar_network_name": names_helper.dynamic_sidecar_network_name, + "simcore_traefik_zone": names_helper.simcore_traefik_zone, + "request_dns": request_dns, + "request_scheme": request_scheme, + "proxy_service_name": names_helper.proxy_service_name, + "request_simcore_user_agent": request_simcore_user_agent, + "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}}, + } + if run_id: + obj_dict["run_id"] = run_id + return cls.parse_obj(obj_dict) + + @classmethod + def from_service_inspect( + cls, service_inspect: Mapping[str, Any] + ) -> "SchedulerData": + labels = service_inspect["Spec"]["Labels"] + return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) + + def as_label_data(self) -> str: + # compose_spec needs to be json encoded before encoding it to json + # and storing it in the label + return self.copy( + update={"compose_spec": json.dumps(self.compose_spec)}, deep=True + ).json() + + class Config: + extra = Extra.allow + allow_population_by_field_name = True diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 322593c9e76..d8f7f356cd7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -5,9 +5,6 @@ from fastapi import FastAPI, status from httpx import AsyncClient -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -24,6 +21,7 @@ ) from servicelib.logging_utils import log_context, log_decorator from servicelib.utils import logged_gather +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import DynamicSidecarSettings from ....modules.dynamic_sidecar.docker_api import get_or_create_networks_ids diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index ce6ccdc1e1b..9f90a555b69 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -12,17 +12,17 @@ DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, DYNAMIC_SIDECAR_SERVICE_PREFIX, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - NetworkId, - SchedulerData, - ServiceId, -) from models_library.docker import to_simcore_runtime_docker_label_key from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from servicelib.json_serialization import json_dumps from servicelib.utils import logged_gather +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + NetworkId, + SchedulerData, + ServiceId, +) from starlette import status from tenacity import TryAgain, retry from tenacity._asyncio import AsyncRetrying diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py index 3639edd6d99..e88a4a8501e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py @@ -1,8 +1,5 @@ from typing import Any -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.docker import StandardSimcoreDockerLabels from models_library.services_resources import ( CPU_10_PERCENT, @@ -11,6 +8,7 @@ MEMORY_250MB, ) from pydantic import ByteSize +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import DynamicSidecarProxySettings, DynamicSidecarSettings from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 1e090bde362..d5e7a3f03a7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -5,9 +5,6 @@ from models_library.api_schemas_directorv2.constants import ( DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import ( StandardSimcoreDockerLabels, @@ -16,6 +13,7 @@ from models_library.service_settings_labels import SimcoreServiceSettingsLabel from pydantic import ByteSize from servicelib.json_serialization import json_dumps +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import AppSettings, DynamicSidecarSettings from .._namespace import get_compose_namespace diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py index 6af5120b401..ae321803352 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py @@ -3,11 +3,11 @@ """ import logging -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DockerContainerInspect, -) from models_library.generated_models.docker_rest_api import ContainerState from models_library.services_enums import ServiceState +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DockerContainerInspect, +) logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py index c3a50abcd48..e6278a69083 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py @@ -3,9 +3,7 @@ from abc import ABC, abstractmethod from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData class DynamicSchedulerEvent(ABC): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index 28ab6566843..8d44b51d459 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -6,13 +6,6 @@ from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DockerContainerInspect, - DockerStatus, - DynamicSidecarStatus, - NetworkId, - SchedulerData, -) from models_library.projects import ProjectAtDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeIDStr @@ -31,6 +24,13 @@ from servicelib.json_serialization import json_dumps from servicelib.rabbitmq import RabbitMQClient from simcore_postgres_database.models.comp_tasks import NodeClass +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + DynamicSidecarStatus, + NetworkId, + SchedulerData, +) from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 1b637e3ede6..e56f136942f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -6,11 +6,6 @@ from typing import Any, Deque, Final from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DockerContainerInspect, - DockerStatus, - SchedulerData, -) from models_library.projects_networks import ProjectsNetworks from models_library.projects_nodes import NodeID from models_library.projects_nodes_io import NodeIDStr @@ -27,6 +22,11 @@ from servicelib.rabbitmq import RabbitMQClient from servicelib.utils import logged_gather from simcore_postgres_database.models.comp_tasks import NodeClass +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + SchedulerData, +) from tenacity import TryAgain from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index 96a3948eb56..b5ace09f00f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -6,12 +6,12 @@ from math import floor from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( +from servicelib.error_codes import create_error_code +from simcore_service_director_v2.models.dynamic_services_scheduler import ( DynamicSidecarStatus, SchedulerData, ServiceName, ) -from servicelib.error_codes import create_error_code from .....core.settings import ( DynamicServicesSchedulerSettings, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 6af778d01c5..c6b5b5d436e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -26,10 +26,6 @@ RetrieveDataOutEnveloped, RunningDynamicServiceDetails, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, - ServiceName, -) from models_library.basic_types import PortInt from models_library.projects import ProjectID from models_library.projects_networks import DockerNetworkAlias @@ -40,6 +36,10 @@ from servicelib.background_task import cancel_task from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + SchedulerData, + ServiceName, +) from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings from ...api_client import SidecarsClient, get_sidecars_client diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index 80ca6697d26..a7b4a45ba81 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -3,16 +3,16 @@ from typing import Final from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DynamicSidecarStatus, - SchedulerData, -) from models_library.api_schemas_directorv2.dynamic_services_service import ( RunningDynamicServiceDetails, ) from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from servicelib.fastapi.long_running_tasks.client import ProgressCallback +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DynamicSidecarStatus, + SchedulerData, +) from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings from ...api_client import SidecarsClient, get_sidecars_client diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 68cd2e7f2c2..c8a8600a99c 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -16,9 +16,6 @@ DYNAMIC_PROXY_SERVICE_PREFIX, DYNAMIC_SIDECAR_SERVICE_PREFIX, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.basic_types import PortInt from models_library.projects import Node, NodesDict from models_library.projects_nodes_io import NodeID @@ -35,6 +32,7 @@ X_SIMCORE_USER_AGENT, ) from simcore_service_director_v2.core.settings import DynamicSidecarSettings +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( DynamicSidecarsScheduler, ) diff --git a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py index 66d3b694af0..66e8729afef 100644 --- a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py +++ b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py @@ -10,12 +10,10 @@ from fastapi import status from httpx import Response from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarNotFoundError, ) diff --git a/services/director-v2/tests/unit/test_models_dynamic_services.py b/services/director-v2/tests/unit/test_models_dynamic_services.py index 633301a4e18..214d83adeea 100644 --- a/services/director-v2/tests/unit/test_models_dynamic_services.py +++ b/services/director-v2/tests/unit/test_models_dynamic_services.py @@ -3,7 +3,7 @@ from collections import namedtuple import pytest -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( +from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, ) from simcore_service_director_v2.models.schemas.dynamic_services import ( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 9bc4d8960ee..41726be30b7 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -4,10 +4,8 @@ from typing import Any -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from simcore_service_director_v2.core.settings import AppSettings +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs.sidecar import ( _get_environment_variables, ) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index a35a3abea1f..9871ac2938a 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -14,14 +14,14 @@ import respx from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DockerContainerInspect, -) from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from simcore_service_director_v2.core.settings import AppSettings +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DockerContainerInspect, +) from simcore_service_director_v2.models.schemas.dynamic_services import ( DynamicSidecarStatus, RunningDynamicServiceDetails, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py index f52311311f9..ec9ef582423 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py @@ -7,16 +7,16 @@ import pytest from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - ContainerState, - DockerContainerInspect, - DockerStatus, -) from pydantic import PositiveFloat, PositiveInt from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from servicelib.exception_utils import _SKIPS_MESSAGE +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + ContainerState, + DockerContainerInspect, + DockerStatus, +) from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.api_client import ( BaseClientHTTPError, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py index d9985b7b80d..1f921d3cb69 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py @@ -13,13 +13,11 @@ import pytest import respx from fastapi import FastAPI -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from respx.router import MockRouter +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.api_client._public import ( SidecarsClient, ) diff --git a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py index 68e34b7a783..8d58d96f675 100644 --- a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py +++ b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py @@ -6,9 +6,7 @@ from pathlib import Path import pytest -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData @pytest.fixture( diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index 2921d8489f6..9f257d37784 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -19,9 +19,6 @@ DynamicServiceCreate, RetrieveDataOutEnveloped, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - SchedulerData, -) from models_library.projects_nodes_io import NodeID from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture @@ -32,6 +29,7 @@ X_DYNAMIC_SIDECAR_REQUEST_SCHEME, X_SIMCORE_USER_AGENT, ) +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.models.schemas.dynamic_services import ( RunningDynamicServiceDetails, ) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index 721abb5de8e..28d0da9a565 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -23,15 +23,15 @@ DYNAMIC_SIDECAR_SERVICE_PREFIX, DYNAMIC_VOLUME_REMOVER_PREFIX, ) -from models_library.api_schemas_directorv2.dynamic_services_scheduler import ( - DockerContainerInspect, - SimcoreServiceLabels, -) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_director_v2.core.settings import DynamicSidecarSettings +from simcore_service_director_v2.models.dynamic_services_scheduler import ( + DockerContainerInspect, + SimcoreServiceLabels, +) from simcore_service_director_v2.models.schemas.dynamic_services import ( SchedulerData, ServiceState, From 0cc69583c433eab5adbea24a138e3f39cd48e76e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:58:29 +0200 Subject: [PATCH 09/20] moved all schemas to models-lib --- .../api_schemas_directorv2}/comp_tasks.py | 0 .../models_library/api_schemas_directorv2}/errors.py | 6 ++---- .../api_schemas_directorv2}/services.py | 0 .../api/routes/computations.py | 12 ++++++------ .../api/routes/computations_tasks.py | 2 +- .../models/domains/comp_tasks.py | 6 +++--- .../models/schemas/__init__.py | 0 .../modules/db/repositories/comp_tasks.py | 5 ++++- .../modules/director_v0.py | 2 +- .../src/simcore_service_director_v2/utils/dask.py | 2 +- .../director-v2/tests/helpers/shared_comp_utils.py | 2 +- .../tests/integration/01/test_computation_api.py | 2 +- .../02/test_dynamic_sidecar_nodeports_integration.py | 2 +- services/director-v2/tests/integration/conftest.py | 2 +- .../tests/unit/test_modules_dask_client.py | 2 +- .../unit/with_dbs/test_api_route_computations.py | 10 +++++----- .../with_dbs/test_api_route_computations_tasks.py | 2 +- .../tests/unit/with_dbs/test_utils_dask.py | 2 +- 18 files changed, 30 insertions(+), 29 deletions(-) rename {services/director-v2/src/simcore_service_director_v2/models/schemas => packages/models-library/src/models_library/api_schemas_directorv2}/comp_tasks.py (100%) rename {services/director-v2/src/simcore_service_director_v2/models/schemas => packages/models-library/src/models_library/api_schemas_directorv2}/errors.py (64%) rename {services/director-v2/src/simcore_service_director_v2/models/schemas => packages/models-library/src/models_library/api_schemas_directorv2}/services.py (100%) delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/schemas/__init__.py diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py similarity index 100% rename from services/director-v2/src/simcore_service_director_v2/models/schemas/comp_tasks.py rename to packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/errors.py b/packages/models-library/src/models_library/api_schemas_directorv2/errors.py similarity index 64% rename from services/director-v2/src/simcore_service_director_v2/models/schemas/errors.py rename to packages/models-library/src/models_library/api_schemas_directorv2/errors.py index 9b04c308f6c..ecf33eefd14 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/errors.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/errors.py @@ -1,15 +1,13 @@ -from typing import List, Optional - from pydantic import BaseModel, Field class Error(BaseModel): - code: Optional[str] = Field(None, description="Server Exception") + code: str | None = Field(None, description="Server Exception") class ErrorType(BaseModel): message: str = Field(..., description="Error message") - errors: Optional[List[Error]] = None + errors: list[Error] | None = None status: int = Field(..., description="Error code") diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/services.py b/packages/models-library/src/models_library/api_schemas_directorv2/services.py similarity index 100% rename from services/director-v2/src/simcore_service_director_v2/models/schemas/services.py rename to packages/models-library/src/models_library/api_schemas_directorv2/services.py diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index 9b9cf1eaa37..1d27623653b 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -20,6 +20,12 @@ import networkx as nx from fastapi import APIRouter, Depends, HTTPException +from models_library.api_schemas_directorv2.comp_tasks import ( + ComputationCreate, + ComputationDelete, + ComputationGet, + ComputationStop, +) from models_library.clusters import DEFAULT_CLUSTER_ID from models_library.projects import ProjectAtDB, ProjectID from models_library.services import ServiceKeyVersion @@ -45,12 +51,6 @@ from ...models.domains.comp_pipelines import CompPipelineAtDB from ...models.domains.comp_runs import CompRunsAtDB from ...models.domains.comp_tasks import CompTaskAtDB -from ...models.schemas.comp_tasks import ( - ComputationCreate, - ComputationDelete, - ComputationGet, - ComputationStop, -) from ...modules.catalog import CatalogClient from ...modules.comp_scheduler.base_scheduler import BaseCompScheduler from ...modules.db.repositories.clusters import ClustersRepository diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py index 9e612decf58..3ad2ff5a86d 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py @@ -11,6 +11,7 @@ import networkx as nx from fastapi import APIRouter, Depends, HTTPException +from models_library.api_schemas_directorv2.comp_tasks import TaskLogFileGet from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID @@ -21,7 +22,6 @@ from ...models.domains.comp_pipelines import CompPipelineAtDB from ...models.domains.comp_tasks import CompTaskAtDB -from ...models.schemas.comp_tasks import TaskLogFileGet from ...modules.db.repositories.comp_pipelines import CompPipelinesRepository from ...modules.db.repositories.comp_tasks import CompTasksRepository from ...utils.dask import get_service_log_file_download_link diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py index 2804795bfcd..2773cb6f009 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py @@ -1,8 +1,9 @@ import datetime from contextlib import suppress -from typing import Any +from typing import Any, ClassVar from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict +from models_library.api_schemas_directorv2.services import NodeRequirements from models_library.basic_regex import VERSION_RE from models_library.errors import ErrorDict from models_library.projects import ProjectID @@ -20,7 +21,6 @@ from simcore_postgres_database.models.comp_tasks import NodeClass, StateType from ...utils.db import DB_TO_RUNNING_STATE, RUNNING_STATE_TO_DB -from ..schemas.services import NodeRequirements class Image(BaseModel): @@ -64,7 +64,7 @@ def migrate_from_requirements(cls, v, values): class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "name": "simcore/services/dynamic/jupyter-octave-python-math", diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/__init__.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py index 033ad525b32..b95b3eab101 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py @@ -7,6 +7,10 @@ import arrow import sqlalchemy as sa from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict +from models_library.api_schemas_directorv2.services import ( + NodeRequirements, + ServiceExtras, +) from models_library.errors import ErrorDict from models_library.function_services_catalog import iter_service_docker_data from models_library.projects import ProjectAtDB, ProjectID @@ -28,7 +32,6 @@ from sqlalchemy.dialects.postgresql import insert from ....models.domains.comp_tasks import CompTaskAtDB, Image, NodeSchema -from ....models.schemas.services import NodeRequirements, ServiceExtras from ....utils.computations import to_node_class from ....utils.db import RUNNING_STATE_TO_DB from ...catalog import CatalogClient, ServiceResourcesDict diff --git a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py index ade533a413e..7e943fbcae3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py @@ -13,6 +13,7 @@ from models_library.api_schemas_directorv2.dynamic_services import ( RunningDynamicServiceDetails, ) +from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.projects import ProjectID from models_library.projects_nodes import NodeID from models_library.service_settings_labels import SimcoreServiceLabels @@ -21,7 +22,6 @@ from servicelib.logging_utils import log_decorator from ..core.settings import DirectorV0Settings -from ..models.schemas.services import ServiceExtras from ..utils.client_decorators import handle_errors, handle_retry from ..utils.clients import unenvelope_or_raise_error diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index ee2542d2f2b..1c97c173cb7 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -20,6 +20,7 @@ ContainerLabelsDict, ) from fastapi import FastAPI +from models_library.api_schemas_directorv2.services import NodeRequirements from models_library.clusters import ClusterID from models_library.docker import StandardSimcoreDockerLabels from models_library.errors import ErrorDict @@ -48,7 +49,6 @@ ) from ..models.domains.comp_runs import MetadataDict from ..models.domains.comp_tasks import Image -from ..models.schemas.services import NodeRequirements from ..modules.osparc_variables_substitutions import ( resolve_and_substitute_session_variables_in_specs, substitute_vendor_secrets_in_specs, diff --git a/services/director-v2/tests/helpers/shared_comp_utils.py b/services/director-v2/tests/helpers/shared_comp_utils.py index 04df1f50979..bd04448854e 100644 --- a/services/director-v2/tests/helpers/shared_comp_utils.py +++ b/services/director-v2/tests/helpers/shared_comp_utils.py @@ -3,6 +3,7 @@ from uuid import UUID import httpx +from models_library.api_schemas_directorv2.comp_tasks import ComputationGet from models_library.clusters import ClusterID from models_library.projects import ProjectAtDB from models_library.projects_pipeline import PipelineDetails @@ -11,7 +12,6 @@ from pydantic import PositiveInt from pydantic.networks import AnyHttpUrl from pytest_simcore.helpers.constants import MINUTE -from simcore_service_director_v2.models.schemas.comp_tasks import ComputationGet from starlette import status from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 32699c7c855..2d70cc4e9ee 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -19,6 +19,7 @@ assert_and_wait_for_pipeline_status, assert_computation_task_out_obj, ) +from models_library.api_schemas_directorv2.comp_tasks import ComputationGet from models_library.clusters import DEFAULT_CLUSTER_ID from models_library.projects import ProjectAtDB from models_library.projects_nodes import NodeState @@ -28,7 +29,6 @@ from models_library.users import UserID from pytest import MonkeyPatch from settings_library.rabbit import RabbitSettings -from simcore_service_director_v2.models.schemas.comp_tasks import ComputationGet from starlette import status from starlette.testclient import TestClient from yarl import URL diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 2c3b764aca8..7b231db9def 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -37,6 +37,7 @@ assert_and_wait_for_pipeline_status, assert_computation_task_out_obj, ) +from models_library.api_schemas_directorv2.comp_tasks import ComputationGet from models_library.api_schemas_directorv2.constants import ( DYNAMIC_SIDECAR_SERVICE_PREFIX, ) @@ -79,7 +80,6 @@ from simcore_sdk.node_data import data_manager from simcore_sdk.node_ports_v2 import DBManager, Nodeports, Port from simcore_service_director_v2.core.settings import AppSettings, RCloneSettings -from simcore_service_director_v2.models.schemas.comp_tasks import ComputationGet from sqlalchemy.dialects.postgresql import insert as pg_insert from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index c05f6d81619..ee576a3df72 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -5,12 +5,12 @@ import httpx import pytest import sqlalchemy as sa +from models_library.api_schemas_directorv2.comp_tasks import ComputationGet from models_library.projects import ProjectAtDB from models_library.users import UserID from pytest_mock import MockerFixture from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.projects import projects -from simcore_service_director_v2.models.schemas.comp_tasks import ComputationGet from starlette import status from tenacity import retry from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 12a7c765437..4eaf5c9ac10 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -43,6 +43,7 @@ from distributed.deploy.spec import SpecCluster from faker import Faker from fastapi.applications import FastAPI +from models_library.api_schemas_directorv2.services import NodeRequirements from models_library.api_schemas_storage import LinkType from models_library.clusters import ClusterID, NoAuthentication, SimpleAuthentication from models_library.docker import to_simcore_runtime_docker_label_key @@ -67,7 +68,6 @@ ) from simcore_service_director_v2.models.domains.comp_runs import MetadataDict from simcore_service_director_v2.models.domains.comp_tasks import Image -from simcore_service_director_v2.models.schemas.services import NodeRequirements from simcore_service_director_v2.modules.dask_client import DaskClient, TaskHandlers from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index 109420b530b..ec9203c1464 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -18,6 +18,11 @@ import respx from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_directorv2.comp_tasks import ( + ComputationCreate, + ComputationGet, +) +from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.basic_types import VersionStr from models_library.clusters import DEFAULT_CLUSTER_ID, Cluster, ClusterID from models_library.projects import ProjectAtDB @@ -40,11 +45,6 @@ from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB -from simcore_service_director_v2.models.schemas.comp_tasks import ( - ComputationCreate, - ComputationGet, -) -from simcore_service_director_v2.models.schemas.services import ServiceExtras from starlette import status pytest_simcore_core_services_selection = ["postgres", "rabbit"] diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py index 9ac4955132a..0b5e7c2f028 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py @@ -10,6 +10,7 @@ import pytest from faker import Faker from fastapi import FastAPI, status +from models_library.api_schemas_directorv2.comp_tasks import TaskLogFileGet from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID @@ -18,7 +19,6 @@ from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB -from simcore_service_director_v2.models.schemas.comp_tasks import TaskLogFileGet pytest_simcore_core_services_selection = [ "postgres", diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py index 79df44cac36..db35cb91ce6 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py @@ -28,6 +28,7 @@ from distributed import SpecCluster from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_directorv2.services import NodeRequirements from models_library.api_schemas_storage import FileUploadLinks, FileUploadSchema from models_library.clusters import ClusterID from models_library.docker import to_simcore_runtime_docker_label_key @@ -42,7 +43,6 @@ from simcore_sdk.node_ports_v2 import FileLinkType from simcore_service_director_v2.models.domains.comp_runs import MetadataDict from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB -from simcore_service_director_v2.models.schemas.services import NodeRequirements from simcore_service_director_v2.modules.dask_clients_pool import DaskClientsPool from simcore_service_director_v2.utils.dask import ( _LOGS_FILE_NAME, From 61eb116ce63bb82184a62b298608846473afcae9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:05:30 +0200 Subject: [PATCH 10/20] last models --- .../api/routes/computations.py | 6 +++--- .../api/routes/computations_tasks.py | 4 ++-- .../simcore_service_director_v2/models/__init__.py | 4 ++++ .../models/{domains => }/comp_pipelines.py | 2 +- .../models/{domains => }/comp_runs.py | 2 +- .../models/{domains => }/comp_tasks.py | 2 +- .../models/domains/__init__.py | 0 .../modules/comp_scheduler/base_scheduler.py | 6 +++--- .../modules/comp_scheduler/dask_scheduler.py | 4 ++-- .../modules/comp_scheduler/factory.py | 2 +- .../modules/dask_client.py | 4 ++-- .../modules/db/repositories/comp_pipelines.py | 2 +- .../modules/db/repositories/comp_runs.py | 2 +- .../modules/db/repositories/comp_tasks.py | 2 +- .../utils/computations.py | 2 +- .../src/simcore_service_director_v2/utils/dags.py | 2 +- .../src/simcore_service_director_v2/utils/dask.py | 4 ++-- services/director-v2/tests/unit/_helpers.py | 6 +++--- .../tests/unit/test_models_comp_pipelines.py | 12 ++++++------ .../director-v2/tests/unit/test_models_comp_runs.py | 8 ++++---- .../director-v2/tests/unit/test_models_comp_tasks.py | 12 ++++++------ .../tests/unit/test_modules_dask_client.py | 4 ++-- .../director-v2/tests/unit/test_utils_computation.py | 7 +++---- services/director-v2/tests/unit/with_dbs/conftest.py | 6 +++--- .../unit/with_dbs/test_api_route_computations.py | 6 +++--- .../with_dbs/test_api_route_computations_tasks.py | 4 ++-- .../test_modules_comp_scheduler_dask_scheduler.py | 6 +++--- .../tests/unit/with_dbs/test_utils_dask.py | 4 ++-- 28 files changed, 64 insertions(+), 61 deletions(-) rename services/director-v2/src/simcore_service_director_v2/models/{domains => }/comp_pipelines.py (98%) rename services/director-v2/src/simcore_service_director_v2/models/{domains => }/comp_runs.py (98%) rename services/director-v2/src/simcore_service_director_v2/models/{domains => }/comp_tasks.py (99%) delete mode 100644 services/director-v2/src/simcore_service_director_v2/models/domains/__init__.py diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index 1d27623653b..0a715a0a734 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -48,9 +48,9 @@ ProjectNotFoundError, SchedulerError, ) -from ...models.domains.comp_pipelines import CompPipelineAtDB -from ...models.domains.comp_runs import CompRunsAtDB -from ...models.domains.comp_tasks import CompTaskAtDB +from ...models.comp_pipelines import CompPipelineAtDB +from ...models.comp_runs import CompRunsAtDB +from ...models.comp_tasks import CompTaskAtDB from ...modules.catalog import CatalogClient from ...modules.comp_scheduler.base_scheduler import BaseCompScheduler from ...modules.db.repositories.clusters import ClustersRepository diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py index 3ad2ff5a86d..f953f66b109 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations_tasks.py @@ -20,8 +20,8 @@ from simcore_sdk.node_ports_v2 import FileLinkType from starlette import status -from ...models.domains.comp_pipelines import CompPipelineAtDB -from ...models.domains.comp_tasks import CompTaskAtDB +from ...models.comp_pipelines import CompPipelineAtDB +from ...models.comp_tasks import CompTaskAtDB from ...modules.db.repositories.comp_pipelines import CompPipelinesRepository from ...modules.db.repositories.comp_tasks import CompTasksRepository from ...utils.dask import get_service_log_file_download_link diff --git a/services/director-v2/src/simcore_service_director_v2/models/__init__.py b/services/director-v2/src/simcore_service_director_v2/models/__init__.py index e69de29bb2d..166628e0516 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/__init__.py +++ b/services/director-v2/src/simcore_service_director_v2/models/__init__.py @@ -0,0 +1,4 @@ +""" +This packages is intended for domain (i.e. internal) models +Any common/base and API (schema) models go in models_library or models_library.api_schemas_directorv2 +""" diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py similarity index 98% rename from services/director-v2/src/simcore_service_director_v2/models/domains/comp_pipelines.py rename to services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py index fd750d0b93f..9d5aaa46a58 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, validator from simcore_postgres_database.models.comp_pipeline import StateType -from ...utils.db import DB_TO_RUNNING_STATE +from ..utils.db import DB_TO_RUNNING_STATE class CompPipelineAtDB(BaseModel): diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py similarity index 98% rename from services/director-v2/src/simcore_service_director_v2/models/domains/comp_runs.py rename to services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index db06bfc52fe..cc7096b897d 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, PositiveInt, validator from simcore_postgres_database.models.comp_pipeline import StateType -from ...utils.db import DB_TO_RUNNING_STATE +from ..utils.db import DB_TO_RUNNING_STATE MetadataDict = dict[str, Any] diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py similarity index 99% rename from services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py rename to services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py index 2773cb6f009..17cd64aa87e 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/domains/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py @@ -20,7 +20,7 @@ from pydantic.types import PositiveInt from simcore_postgres_database.models.comp_tasks import NodeClass, StateType -from ...utils.db import DB_TO_RUNNING_STATE, RUNNING_STATE_TO_DB +from ..utils.db import DB_TO_RUNNING_STATE, RUNNING_STATE_TO_DB class Image(BaseModel): diff --git a/services/director-v2/src/simcore_service_director_v2/models/domains/__init__.py b/services/director-v2/src/simcore_service_director_v2/models/domains/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py index 1138e67386b..72223dfaffb 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py @@ -39,9 +39,9 @@ SchedulerError, TaskSchedulingError, ) -from ...models.domains.comp_pipelines import CompPipelineAtDB -from ...models.domains.comp_runs import CompRunsAtDB, MetadataDict -from ...models.domains.comp_tasks import CompTaskAtDB, Image +from ...models.comp_pipelines import CompPipelineAtDB +from ...models.comp_runs import CompRunsAtDB, MetadataDict +from ...models.comp_tasks import CompTaskAtDB, Image from ...utils.computations import get_pipeline_state_from_task_states from ...utils.scheduler import ( COMPLETED_STATES, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py index 873f387c78f..a032a4731d0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py @@ -24,11 +24,11 @@ from models_library.users import UserID from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.comp_tasks import NodeClass -from simcore_service_director_v2.models.domains.comp_runs import MetadataDict +from simcore_service_director_v2.models.comp_runs import MetadataDict from ...core.errors import TaskSchedulingError from ...core.settings import ComputationalBackendSettings -from ...models.domains.comp_tasks import CompTaskAtDB, Image +from ...models.comp_tasks import CompTaskAtDB, Image from ...modules.dask_client import DaskClient from ...modules.dask_clients_pool import DaskClientsPool from ...modules.db.repositories.clusters import ClustersRepository diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py index 60a4d7a7cf5..192a2dee832 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/factory.py @@ -4,7 +4,7 @@ from models_library.clusters import DEFAULT_CLUSTER_ID from ...core.errors import ConfigurationError -from ...models.domains.comp_runs import CompRunsAtDB +from ...models.comp_runs import CompRunsAtDB from ...modules.dask_clients_pool import DaskClientsPool from ...modules.rabbitmq import get_rabbitmq_client from ...utils.scheduler import SCHEDULED_STATES diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index dbdf458e86d..e97fbfa926e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -60,8 +60,8 @@ ComputationalBackendTaskResultsNotReadyError, ) from ..core.settings import AppSettings, ComputationalBackendSettings -from ..models.domains.comp_runs import MetadataDict -from ..models.domains.comp_tasks import Image +from ..models.comp_runs import MetadataDict +from ..models.comp_tasks import Image from ..modules.storage import StorageClient from ..utils.dask import ( check_communication_with_scheduler_is_open, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py index 7af66309154..3c24694c2fd 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py @@ -8,7 +8,7 @@ from sqlalchemy.dialects.postgresql import insert from ....core.errors import PipelineNotFoundError -from ....models.domains.comp_pipelines import CompPipelineAtDB +from ....models.comp_pipelines import CompPipelineAtDB from ..tables import comp_pipeline from ._base import BaseRepository diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py index 5e0725ce1d6..80d63369c7a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py @@ -17,7 +17,7 @@ from sqlalchemy.sql.expression import desc from ....core.errors import ClusterNotFoundError, ComputationalRunNotFoundError -from ....models.domains.comp_runs import CompRunsAtDB, MetadataDict +from ....models.comp_runs import CompRunsAtDB, MetadataDict from ....utils.db import RUNNING_STATE_TO_DB from ..tables import comp_runs from ._base import BaseRepository diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py index b95b3eab101..f9198ec7740 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks.py @@ -31,7 +31,7 @@ from sqlalchemy import literal_column from sqlalchemy.dialects.postgresql import insert -from ....models.domains.comp_tasks import CompTaskAtDB, Image, NodeSchema +from ....models.comp_tasks import CompTaskAtDB, Image, NodeSchema from ....utils.computations import to_node_class from ....utils.db import RUNNING_STATE_TO_DB from ...catalog import CatalogClient, ServiceResourcesDict diff --git a/services/director-v2/src/simcore_service_director_v2/utils/computations.py b/services/director-v2/src/simcore_service_director_v2/utils/computations.py index 7e473bcc0b6..febdaaf8b48 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/computations.py @@ -8,7 +8,7 @@ from pydantic import parse_obj_as from servicelib.utils import logged_gather -from ..models.domains.comp_tasks import CompTaskAtDB +from ..models.comp_tasks import CompTaskAtDB from ..modules.catalog import CatalogClient from ..modules.db.tables import NodeClass diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dags.py b/services/director-v2/src/simcore_service_director_v2/utils/dags.py index 69ded8407dc..5676e2e7a0e 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dags.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dags.py @@ -12,7 +12,7 @@ from models_library.projects_state import RunningState from models_library.utils.nodes import compute_node_hash -from ..models.domains.comp_tasks import CompTaskAtDB +from ..models.comp_tasks import CompTaskAtDB from ..modules.db.tables import NodeClass from .computations import to_node_class diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 1c97c173cb7..61a4db58b67 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -47,8 +47,8 @@ MissingComputationalResourcesError, PortsValidationError, ) -from ..models.domains.comp_runs import MetadataDict -from ..models.domains.comp_tasks import Image +from ..models.comp_runs import MetadataDict +from ..models.comp_tasks import Image from ..modules.osparc_variables_substitutions import ( resolve_and_substitute_session_variables_in_specs, substitute_vendor_secrets_in_specs, diff --git a/services/director-v2/tests/unit/_helpers.py b/services/director-v2/tests/unit/_helpers.py index b8b333e9d07..60eea950f12 100644 --- a/services/director-v2/tests/unit/_helpers.py +++ b/services/director-v2/tests/unit/_helpers.py @@ -8,9 +8,9 @@ from models_library.projects_nodes_io import NodeID from simcore_postgres_database.models.comp_pipeline import StateType from simcore_postgres_database.models.comp_tasks import comp_tasks -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB -from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB from simcore_service_director_v2.modules.comp_scheduler.base_scheduler import ( BaseCompScheduler, ) diff --git a/services/director-v2/tests/unit/test_models_comp_pipelines.py b/services/director-v2/tests/unit/test_models_comp_pipelines.py index 642ff40ff22..13e8110ca81 100644 --- a/services/director-v2/tests/unit/test_models_comp_pipelines.py +++ b/services/director-v2/tests/unit/test_models_comp_pipelines.py @@ -4,14 +4,14 @@ from copy import deepcopy from pprint import pformat -from typing import Any, Dict, Type +from typing import Any from uuid import UUID import networkx as nx import pytest from models_library.projects_state import RunningState from pydantic.main import BaseModel -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB @pytest.mark.parametrize( @@ -19,7 +19,7 @@ (CompPipelineAtDB,), ) def test_computation_pipeline_model_examples( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) @@ -32,7 +32,7 @@ def test_computation_pipeline_model_examples( (CompPipelineAtDB,), ) def test_computation_pipeline_model_with_running_state_value_field( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): example["state"] = RunningState.RETRY.value # this is a specific Runningstate @@ -46,7 +46,7 @@ def test_computation_pipeline_model_with_running_state_value_field( (CompPipelineAtDB,), ) def test_computation_pipeline_model_with_uuids_in_dag_field( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): old_dag_list = deepcopy(example["dag_adjacency_list"]) @@ -63,7 +63,7 @@ def test_computation_pipeline_model_with_uuids_in_dag_field( (CompPipelineAtDB,), ) def test_computation_pipeline_model_get_graph( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) diff --git a/services/director-v2/tests/unit/test_models_comp_runs.py b/services/director-v2/tests/unit/test_models_comp_runs.py index 078e2b26be9..6dfaddf8b3e 100644 --- a/services/director-v2/tests/unit/test_models_comp_runs.py +++ b/services/director-v2/tests/unit/test_models_comp_runs.py @@ -3,12 +3,12 @@ # pylint:disable=redefined-outer-name from pprint import pformat -from typing import Any, Dict, Type +from typing import Any import pytest from models_library.projects_state import RunningState from pydantic.main import BaseModel -from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB @pytest.mark.parametrize( @@ -16,7 +16,7 @@ (CompRunsAtDB,), ) def test_computation_run_model_examples( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) @@ -29,7 +29,7 @@ def test_computation_run_model_examples( (CompRunsAtDB,), ) def test_computation_run_model_with_run_result_value_field( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): example["result"] = RunningState.RETRY.value diff --git a/services/director-v2/tests/unit/test_models_comp_tasks.py b/services/director-v2/tests/unit/test_models_comp_tasks.py index 54e85d1a3b8..59a19906e5c 100644 --- a/services/director-v2/tests/unit/test_models_comp_tasks.py +++ b/services/director-v2/tests/unit/test_models_comp_tasks.py @@ -3,13 +3,13 @@ # pylint:disable=redefined-outer-name from pprint import pformat -from typing import Any, Dict, Type +from typing import Any import pytest from models_library.projects_state import RunningState from pydantic.main import BaseModel from simcore_postgres_database.models.comp_pipeline import StateType -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB @pytest.mark.parametrize( @@ -17,7 +17,7 @@ (CompTaskAtDB,), ) def test_computation_task_model_examples( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) @@ -30,7 +30,7 @@ def test_computation_task_model_examples( (CompTaskAtDB,), ) def test_computation_task_model_export_to_db_model( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) @@ -49,7 +49,7 @@ def test_computation_task_model_export_to_db_model( (CompTaskAtDB,), ) def test_computation_task_model_with_running_state_value_field( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): example["state"] = RunningState.RETRY.value @@ -63,7 +63,7 @@ def test_computation_task_model_with_running_state_value_field( (CompTaskAtDB,), ) def test_computation_task_model_with_wrong_default_value_field( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for name, example in model_cls_examples.items(): for output_schema in example.get("schema", {}).get("outputs", {}).values(): diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 4eaf5c9ac10..82599c6840c 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -66,8 +66,8 @@ InsuficientComputationalResourcesError, MissingComputationalResourcesError, ) -from simcore_service_director_v2.models.domains.comp_runs import MetadataDict -from simcore_service_director_v2.models.domains.comp_tasks import Image +from simcore_service_director_v2.models.comp_runs import MetadataDict +from simcore_service_director_v2.models.comp_tasks import Image from simcore_service_director_v2.modules.dask_client import DaskClient, TaskHandlers from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/tests/unit/test_utils_computation.py b/services/director-v2/tests/unit/test_utils_computation.py index 1f55d85ab26..dfcd7d26a1b 100644 --- a/services/director-v2/tests/unit/test_utils_computation.py +++ b/services/director-v2/tests/unit/test_utils_computation.py @@ -4,12 +4,11 @@ # pylint:disable=protected-access from pathlib import Path -from typing import List import faker import pytest from models_library.projects_state import RunningState -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB from simcore_service_director_v2.utils.computations import ( get_pipeline_state_from_task_states, is_pipeline_running, @@ -253,11 +252,11 @@ def fake_task(fake_task_file: Path) -> CompTaskAtDB: ], ) def test_get_pipeline_state_from_task_states( - task_states: List[RunningState], + task_states: list[RunningState], exp_pipeline_state: RunningState, fake_task: CompTaskAtDB, ): - tasks: List[CompTaskAtDB] = [ + tasks: list[CompTaskAtDB] = [ fake_task.copy(deep=True, update={"state": s}) for s in task_states ] diff --git a/services/director-v2/tests/unit/with_dbs/conftest.py b/services/director-v2/tests/unit/with_dbs/conftest.py index a1a03a37bd7..50a8bf77f6e 100644 --- a/services/director-v2/tests/unit/with_dbs/conftest.py +++ b/services/director-v2/tests/unit/with_dbs/conftest.py @@ -22,9 +22,9 @@ from simcore_postgres_database.models.comp_pipeline import StateType, comp_pipeline from simcore_postgres_database.models.comp_runs import comp_runs from simcore_postgres_database.models.comp_tasks import comp_tasks -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB -from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB, Image +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB, Image from simcore_service_director_v2.utils.computations import to_node_class from simcore_service_director_v2.utils.dask import generate_dask_job_id from simcore_service_director_v2.utils.db import to_clusters_db diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index ec9203c1464..2c329826c4f 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -42,9 +42,9 @@ from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.comp_pipeline import StateType from simcore_postgres_database.models.comp_tasks import NodeClass -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB -from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB from starlette import status pytest_simcore_core_services_selection = ["postgres", "rabbit"] diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py index 0b5e7c2f028..de286d85b56 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py @@ -17,8 +17,8 @@ from pydantic import parse_raw_as from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB pytest_simcore_core_services_selection = [ "postgres", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index ba5bbc21639..ffb664a2173 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -46,9 +46,9 @@ SchedulerError, ) from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.domains.comp_pipelines import CompPipelineAtDB -from simcore_service_director_v2.models.domains.comp_runs import CompRunsAtDB -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB from simcore_service_director_v2.modules.comp_scheduler import background_task from simcore_service_director_v2.modules.comp_scheduler.base_scheduler import ( BaseCompScheduler, diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py index db35cb91ce6..4a5178272b3 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py @@ -41,8 +41,8 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_sdk.node_ports_v2 import FileLinkType -from simcore_service_director_v2.models.domains.comp_runs import MetadataDict -from simcore_service_director_v2.models.domains.comp_tasks import CompTaskAtDB +from simcore_service_director_v2.models.comp_runs import MetadataDict +from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB from simcore_service_director_v2.modules.dask_clients_pool import DaskClientsPool from simcore_service_director_v2.utils.dask import ( _LOGS_FILE_NAME, From 9f388fecbe40a4f75cdf7ab5d337ffd55eaa5fc5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:09:30 +0200 Subject: [PATCH 11/20] mionr --- .../catalog/src/simcore_service_catalog/models/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/catalog/src/simcore_service_catalog/models/__init__.py b/services/catalog/src/simcore_service_catalog/models/__init__.py index e69de29bb2d..d699149a31c 100644 --- a/services/catalog/src/simcore_service_catalog/models/__init__.py +++ b/services/catalog/src/simcore_service_catalog/models/__init__.py @@ -0,0 +1,4 @@ +""" +This packages is intended for domain (i.e. internal) models +Any common/base and API (schema) models go in models_library or models_library.api_schemas_catalog +""" From 678f99fc4dad4f897ebeb5454b206d68675df12c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:09:39 +0200 Subject: [PATCH 12/20] minor --- .../src/simcore_service_director_v2/models/comp_pipelines.py | 3 ++- .../src/simcore_service_director_v2/models/comp_runs.py | 4 ++-- .../src/simcore_service_director_v2/models/comp_tasks.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py index 9d5aaa46a58..22bac75b1cb 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py @@ -1,4 +1,5 @@ from contextlib import suppress +from typing import Any, ClassVar import networkx as nx from models_library.projects import ProjectID @@ -41,7 +42,7 @@ def get_graph(self) -> nx.DiGraph: class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # DB model { diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index cc7096b897d..c5f64120ce4 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -1,6 +1,6 @@ import datetime from contextlib import suppress -from typing import Any +from typing import Any, ClassVar from models_library.clusters import DEFAULT_CLUSTER_ID, ClusterID from models_library.projects import ProjectID @@ -62,7 +62,7 @@ def ensure_utc(cls, v: datetime.datetime | None) -> datetime.datetime | None: class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # DB model { diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py index 17cd64aa87e..b93d3baf1a1 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py @@ -162,7 +162,7 @@ def to_db_model(self, **exclusion_rules) -> dict[str, Any]: class Config: extra = Extra.forbid orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # DB model { From 3346645a7948a0f9acb770d1df55346656ce2a26 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:13:46 +0200 Subject: [PATCH 13/20] rm dependencies --- .../api_schemas_directorv2/comp_tasks.py | 5 ++- .../dynamic_services_service.py | 23 ---------- .../models/dynamic_services_scheduler.py | 3 +- .../scheduler/_core/_scheduler_utils.py | 42 ++++++++++++++----- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py index 54cd1f4f1f9..146817229c4 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py @@ -40,7 +40,8 @@ class ComputationCreate(BaseModel): @classmethod def ensure_product_name_defined_if_computation_starts(cls, v, values): if "start_pipeline" in values and values["start_pipeline"] and v is None: - raise ValueError("product_name must be set if computation shall start!") + msg = "product_name must be set if computation shall start!" + raise ValueError(msg) return v @@ -50,7 +51,7 @@ class ComputationStop(BaseModel): class ComputationDelete(ComputationStop): force: bool | None = Field( - False, + default=False, description="if True then the pipeline will be removed even if it is running", ) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py index a3bed72aa76..4a413466c30 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py @@ -95,29 +95,6 @@ class RunningDynamicServiceDetails(ServiceDetails): def legacy_service_url(self) -> str: return f"http://{self.host}:{self.internal_port}{self.basepath}" # NOSONAR - @classmethod - def from_scheduler_data( - cls, - node_uuid: NodeID, - scheduler_data: "SchedulerData", # type: ignore - service_state: ServiceState, - service_message: str, - ) -> "RunningDynamicServiceDetails": - return cls.parse_obj( - { - "boot_type": ServiceBootType.V2, - "user_id": scheduler_data.user_id, - "project_id": scheduler_data.project_id, - "service_uuid": node_uuid, - "service_key": scheduler_data.key, - "service_version": scheduler_data.version, - "service_host": scheduler_data.service_name, - "service_port": scheduler_data.service_port, - "service_state": service_state.value, - "service_message": service_message, - } - ) - class Config(ServiceDetails.Config): keep_untouched = (cached_property,) schema_extra: ClassVar[dict[str, Any]] = { diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 944bc6c0d8c..5032163dfb7 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -14,6 +14,7 @@ REGEX_DY_SERVICE_PROXY, REGEX_DY_SERVICE_SIDECAR, ) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.api_schemas_directorv2.dynamic_services_service import ( CommonServiceDetails, ) @@ -428,7 +429,7 @@ def get_proxy_endpoint(self) -> AnyHttpUrl: def from_http_request( # pylint: disable=too-many-arguments cls, - service: "DynamicServiceCreate", # type: ignore + service: DynamicServiceCreate, simcore_service_labels: SimcoreServiceLabels, port: PortInt, request_dns: str, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index a7b4a45ba81..aede853010d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -7,7 +7,7 @@ RunningDynamicServiceDetails, ) from models_library.projects_nodes_io import NodeID -from models_library.services_enums import ServiceState +from models_library.services_enums import ServiceBootType, ServiceState from servicelib.fastapi.long_running_tasks.client import ProgressCallback from simcore_service_director_v2.models.dynamic_services_scheduler import ( DynamicSidecarStatus, @@ -90,10 +90,10 @@ async def cleanup_volume_removal_services(app: FastAPI) -> None: ) -async def discover_running_services(schduler: "Scheduler") -> None: # type: ignore +async def discover_running_services(scheduler: "Scheduler") -> None: # type: ignore """discover all services which were started before and add them to the scheduler""" dynamic_sidecar_settings: DynamicSidecarSettings = ( - schduler.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR + scheduler.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR ) services_to_observe: list[SchedulerData] = await get_dynamic_sidecars_to_observe( dynamic_sidecar_settings @@ -102,7 +102,29 @@ async def discover_running_services(schduler: "Scheduler") -> None: # type: ign logger.info("The following services need to be observed: %s", services_to_observe) for scheduler_data in services_to_observe: - await schduler._add_service(scheduler_data) # pylint: disable=protected-access + await scheduler._add_service(scheduler_data) # pylint: disable=protected-access + + +def _create_model_from_scheduler_data( + node_uuid: NodeID, + scheduler_data: "SchedulerData", # type: ignore + service_state: ServiceState, + service_message: str, +) -> RunningDynamicServiceDetails: + return RunningDynamicServiceDetails.parse_obj( + { + "boot_type": ServiceBootType.V2, + "user_id": scheduler_data.user_id, + "project_id": scheduler_data.project_id, + "service_uuid": node_uuid, + "service_key": scheduler_data.key, + "service_version": scheduler_data.version, + "service_host": scheduler_data.service_name, + "service_port": scheduler_data.service_port, + "service_state": service_state.value, + "service_message": service_message, + } + ) async def get_stack_status_from_scheduler_data( @@ -113,7 +135,7 @@ async def get_stack_status_from_scheduler_data( # check if there was an error picked up by the scheduler # and marked this service as failed if scheduler_data.dynamic_sidecar.status.current != DynamicSidecarStatus.OK: - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.FAILED, @@ -122,7 +144,7 @@ async def get_stack_status_from_scheduler_data( # is the service stopping? if scheduler_data.dynamic_sidecar.service_removal_state.can_remove: - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.STOPPING, @@ -138,7 +160,7 @@ async def get_stack_status_from_scheduler_data( ) except DockerServiceNotFoundError: # in this case, the service is starting, so state is pending - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.PENDING, @@ -147,7 +169,7 @@ async def get_stack_status_from_scheduler_data( # while the dynamic-sidecar state is not RUNNING report it's state if sidecar_state != ServiceState.RUNNING: - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=sidecar_state, @@ -160,7 +182,7 @@ async def get_stack_status_from_scheduler_data( # wait for containers to start if len(scheduler_data.dynamic_sidecar.containers_inspect) == 0: # marks status as waiting for containers - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.STARTING, @@ -171,7 +193,7 @@ async def get_stack_status_from_scheduler_data( container_state, container_message = extract_containers_minimum_statuses( scheduler_data.dynamic_sidecar.containers_inspect ) - return RunningDynamicServiceDetails.from_scheduler_data( + return _create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=container_state, From 983f9020685e9502c402380d858bf65a6dc2e2f6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:24:10 +0200 Subject: [PATCH 14/20] mv constant --- .../src/simcore_service_director_v2}/constants.py | 0 .../simcore_service_director_v2/core/settings.py | 4 +--- .../models/dynamic_services_scheduler.py | 14 +++++++------- .../modules/dynamic_sidecar/_namespace.py | 4 +--- .../modules/dynamic_sidecar/docker_api/_core.py | 8 ++++---- .../modules/dynamic_sidecar/docker_api/_volume.py | 4 +--- .../docker_service_specs/sidecar.py | 4 +--- .../docker_service_specs/volume_remover.py | 4 +--- .../test_dynamic_sidecar_nodeports_integration.py | 4 +--- services/director-v2/tests/integration/02/utils.py | 8 ++++---- services/director-v2/tests/unit/conftest.py | 4 +--- .../test_modules_dynamic_sidecar_docker_api.py | 10 +++++----- 12 files changed, 27 insertions(+), 41 deletions(-) rename {packages/models-library/src/models_library/api_schemas_directorv2 => services/director-v2/src/simcore_service_director_v2}/constants.py (100%) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/constants.py b/services/director-v2/src/simcore_service_director_v2/constants.py similarity index 100% rename from packages/models-library/src/models_library/api_schemas_directorv2/constants.py rename to services/director-v2/src/simcore_service_director_v2/constants.py diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index e1ca1784f15..ebcce949771 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -9,9 +9,6 @@ from functools import cached_property from pathlib import Path -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_DOCKER_IMAGE_RE, -) from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -50,6 +47,7 @@ from settings_library.utils_service import DEFAULT_FASTAPI_PORT from simcore_postgres_database.models.clusters import ClusterType from simcore_sdk.node_ports_v2 import FileLinkType +from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_DOCKER_IMAGE_RE logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 5032163dfb7..387baf04358 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -7,13 +7,6 @@ from typing import Any, TypeAlias from uuid import UUID -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_PROXY_SERVICE_PREFIX, - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, - REGEX_DY_SERVICE_PROXY, - REGEX_DY_SERVICE_SIDECAR, -) from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.api_schemas_directorv2.dynamic_services_service import ( CommonServiceDetails, @@ -31,6 +24,13 @@ from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as from servicelib.error_codes import ErrorCodeStr from servicelib.exception_utils import DelayedExceptionHandler +from simcore_service_director_v2.constants import ( + DYNAMIC_PROXY_SERVICE_PREFIX, + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, + REGEX_DY_SERVICE_PROXY, + REGEX_DY_SERVICE_SIDECAR, +) TEMPORARY_PORT_NUMBER = 65_534 diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py index 154cd9a0600..4ca61d0ea00 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py @@ -1,7 +1,5 @@ -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) from models_library.projects_nodes import NodeID +from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX def get_compose_namespace(node_uuid: NodeID) -> str: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index 9f90a555b69..7cca46b0f6e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -8,16 +8,16 @@ from aiodocker.utils import clean_filters, clean_map from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) from models_library.docker import to_simcore_runtime_docker_label_key from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from servicelib.json_serialization import json_dumps from servicelib.utils import logged_gather +from simcore_service_director_v2.constants import ( + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) from simcore_service_director_v2.models.dynamic_services_scheduler import ( NetworkId, SchedulerData, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py index 59d21c90471..3255449d3d5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py @@ -2,14 +2,12 @@ from datetime import datetime, timezone from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_VOLUME_REMOVER_PREFIX, -) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID from servicelib.docker_utils import to_datetime from servicelib.logging_utils import log_context +from simcore_service_director_v2.constants import DYNAMIC_VOLUME_REMOVER_PREFIX from tenacity import TryAgain from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index d5e7a3f03a7..e4874b366d8 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -2,9 +2,6 @@ from copy import deepcopy from models_library.aiodocker_api import AioDockerServiceSpec -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, -) from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import ( StandardSimcoreDockerLabels, @@ -13,6 +10,7 @@ from models_library.service_settings_labels import SimcoreServiceSettingsLabel from pydantic import ByteSize from servicelib.json_serialization import json_dumps +from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import AppSettings, DynamicSidecarSettings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py index 9d7b80c3cef..f59708d0c6a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py @@ -5,9 +5,6 @@ from uuid import uuid4 from models_library.aiodocker_api import AioDockerServiceSpec -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_VOLUME_REMOVER_PREFIX, -) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services_resources import ( @@ -18,6 +15,7 @@ ) from models_library.users import UserID from pydantic import parse_obj_as +from simcore_service_director_v2.constants import DYNAMIC_VOLUME_REMOVER_PREFIX from ....core.settings import DynamicSidecarSettings diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 7b231db9def..8a97e51ee95 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -38,9 +38,6 @@ assert_computation_task_out_obj, ) from models_library.api_schemas_directorv2.comp_tasks import ComputationGet -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) from models_library.clusters import DEFAULT_CLUSTER_ID from models_library.projects import ( Node, @@ -79,6 +76,7 @@ from simcore_sdk import node_ports_v2 from simcore_sdk.node_data import data_manager from simcore_sdk.node_ports_v2 import DBManager, Nodeports, Port +from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX from simcore_service_director_v2.core.settings import AppSettings, RCloneSettings from sqlalchemy.dialects.postgresql import insert as pg_insert from tenacity._asyncio import AsyncRetrying diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index c8a8600a99c..97ea99765a2 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -12,10 +12,6 @@ import aiodocker import httpx from fastapi import FastAPI -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_PROXY_SERVICE_PREFIX, - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) from models_library.basic_types import PortInt from models_library.projects import Node, NodesDict from models_library.projects_nodes_io import NodeID @@ -31,6 +27,10 @@ X_DYNAMIC_SIDECAR_REQUEST_SCHEME, X_SIMCORE_USER_AGENT, ) +from simcore_service_director_v2.constants import ( + DYNAMIC_PROXY_SERVICE_PREFIX, + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) from simcore_service_director_v2.core.settings import DynamicSidecarSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index 26c63f6abce..e5023988a6e 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -27,9 +27,6 @@ from distributed.deploy.spec import SpecCluster from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_directorv2.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, -) from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.basic_types import PortInt from models_library.clusters import ClusterID @@ -44,6 +41,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.s3 import S3Settings from simcore_sdk.node_ports_v2 import FileLinkType +from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.schemas.dynamic_services import ( SchedulerData, diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index 28d0da9a565..462a60d1589 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -17,16 +17,16 @@ from aiodocker.volumes import DockerVolume from faker import Faker from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_directorv2.constants import ( +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from models_library.users import UserID +from pytest_simcore.helpers.utils_envs import EnvVarsDict +from simcore_service_director_v2.constants import ( DYNAMIC_PROXY_SERVICE_PREFIX, DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, DYNAMIC_SIDECAR_SERVICE_PREFIX, DYNAMIC_VOLUME_REMOVER_PREFIX, ) -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.users import UserID -from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_director_v2.core.settings import DynamicSidecarSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, From 9d46d6d0a79735e43b31a590517ff565fe973829 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:28:20 +0200 Subject: [PATCH 15/20] relative import --- .../src/simcore_service_director_v2/cli/_core.py | 4 +--- .../simcore_service_director_v2/core/settings.py | 3 ++- .../models/dynamic_services_scheduler.py | 3 ++- .../modules/comp_scheduler/dask_scheduler.py | 2 +- .../modules/dynamic_sidecar/_namespace.py | 3 ++- .../modules/dynamic_sidecar/api_client/_public.py | 2 +- .../modules/dynamic_sidecar/docker_api/_core.py | 14 +++++--------- .../modules/dynamic_sidecar/docker_api/_volume.py | 2 +- .../dynamic_sidecar/docker_service_specs/proxy.py | 2 +- .../docker_service_specs/sidecar.py | 4 ++-- .../docker_service_specs/volume_remover.py | 2 +- .../modules/dynamic_sidecar/docker_states.py | 5 ++--- .../dynamic_sidecar/scheduler/_core/_abc.py | 3 ++- .../dynamic_sidecar/scheduler/_core/_events.py | 14 +++++++------- .../scheduler/_core/_events_utils.py | 10 +++++----- .../dynamic_sidecar/scheduler/_core/_observer.py | 10 +++++----- .../dynamic_sidecar/scheduler/_core/_scheduler.py | 5 +---- .../scheduler/_core/_scheduler_utils.py | 5 +---- 18 files changed, 42 insertions(+), 51 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_core.py b/services/director-v2/src/simcore_service_director_v2/cli/_core.py index 81b910c370c..23a7fe244a9 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_core.py @@ -16,15 +16,13 @@ from rich.live import Live from rich.table import Table from servicelib.services_utils import get_service_from_key -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DynamicSidecarNamesHelper, -) from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_attempt from tenacity.wait import wait_random_exponential from ..core.application import create_base_app from ..core.settings import AppSettings +from ..models.dynamic_services_scheduler import DynamicSidecarNamesHelper from ..modules import db, director_v0, dynamic_sidecar from ..modules.db.repositories.projects import ProjectsRepository from ..modules.director_v0 import DirectorV0Client diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index ebcce949771..619ee936172 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -47,7 +47,8 @@ from settings_library.utils_service import DEFAULT_FASTAPI_PORT from simcore_postgres_database.models.clusters import ClusterType from simcore_sdk.node_ports_v2 import FileLinkType -from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_DOCKER_IMAGE_RE + +from ..constants import DYNAMIC_SIDECAR_DOCKER_IMAGE_RE logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 387baf04358..cdf06f67cae 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -24,7 +24,8 @@ from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as from servicelib.error_codes import ErrorCodeStr from servicelib.exception_utils import DelayedExceptionHandler -from simcore_service_director_v2.constants import ( + +from ..constants import ( DYNAMIC_PROXY_SERVICE_PREFIX, DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, DYNAMIC_SIDECAR_SERVICE_PREFIX, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py index a032a4731d0..11e9a0d4b54 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py @@ -24,7 +24,6 @@ from models_library.users import UserID from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.comp_tasks import NodeClass -from simcore_service_director_v2.models.comp_runs import MetadataDict from ...core.errors import TaskSchedulingError from ...core.settings import ComputationalBackendSettings @@ -39,6 +38,7 @@ ) from ...utils.dask_client_utils import TaskHandlers from ..db.repositories.comp_tasks import CompTasksRepository +from ..models.comp_runs import MetadataDict from .base_scheduler import BaseCompScheduler logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py index 4ca61d0ea00..0d62cee0b85 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namespace.py @@ -1,5 +1,6 @@ from models_library.projects_nodes import NodeID -from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX + +from ...constants import DYNAMIC_SIDECAR_SERVICE_PREFIX def get_compose_namespace(node_uuid: NodeID) -> str: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index d8f7f356cd7..a6ac1cd7985 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -21,9 +21,9 @@ ) from servicelib.logging_utils import log_context, log_decorator from servicelib.utils import logged_gather -from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import DynamicSidecarSettings +from ....models.dynamic_services_scheduler import SchedulerData from ....modules.dynamic_sidecar.docker_api import get_or_create_networks_ids from ..errors import EntrypointContainerNotFoundError from ._errors import BaseClientHTTPError, UnexpectedStatusError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index 7cca46b0f6e..c56f24ffdcb 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -14,15 +14,6 @@ from models_library.services_enums import ServiceState from servicelib.json_serialization import json_dumps from servicelib.utils import logged_gather -from simcore_service_director_v2.constants import ( - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, - DYNAMIC_SIDECAR_SERVICE_PREFIX, -) -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - NetworkId, - SchedulerData, - ServiceId, -) from starlette import status from tenacity import TryAgain, retry from tenacity._asyncio import AsyncRetrying @@ -30,7 +21,12 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_exponential, wait_random_exponential +from ....constants import ( + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL, + DYNAMIC_SIDECAR_SERVICE_PREFIX, +) from ....core.settings import DynamicSidecarSettings +from ....models.dynamic_services_scheduler import NetworkId, SchedulerData, ServiceId from ....utils.dict_utils import get_leaf_key_paths, nested_update from ..docker_states import TASK_STATES_RUNNING, extract_task_state from ..errors import DockerServiceNotFoundError, DynamicSidecarError, GenericDockerError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py index 3255449d3d5..620378210ee 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_volume.py @@ -7,13 +7,13 @@ from models_library.users import UserID from servicelib.docker_utils import to_datetime from servicelib.logging_utils import log_context -from simcore_service_director_v2.constants import DYNAMIC_VOLUME_REMOVER_PREFIX from tenacity import TryAgain from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +from ....constants import DYNAMIC_VOLUME_REMOVER_PREFIX from ....core.settings import DynamicSidecarSettings from ..docker_service_specs.volume_remover import spec_volume_removal_service from ._utils import docker_client diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py index e88a4a8501e..c99a421cc0b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/proxy.py @@ -8,9 +8,9 @@ MEMORY_250MB, ) from pydantic import ByteSize -from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from ....core.settings import DynamicSidecarProxySettings, DynamicSidecarSettings +from ....models.dynamic_services_scheduler import SchedulerData from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index e4874b366d8..13958084a89 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -10,10 +10,10 @@ from models_library.service_settings_labels import SimcoreServiceSettingsLabel from pydantic import ByteSize from servicelib.json_serialization import json_dumps -from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL -from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData +from ....constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL from ....core.settings import AppSettings, DynamicSidecarSettings +from ....models.dynamic_services_scheduler import SchedulerData from .._namespace import get_compose_namespace from ..volumes import DynamicSidecarVolumesPathsResolver from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py index f59708d0c6a..bd351cb87c4 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/volume_remover.py @@ -15,8 +15,8 @@ ) from models_library.users import UserID from pydantic import parse_obj_as -from simcore_service_director_v2.constants import DYNAMIC_VOLUME_REMOVER_PREFIX +from ....constants import DYNAMIC_VOLUME_REMOVER_PREFIX from ....core.settings import DynamicSidecarSettings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py index ae321803352..5a503f8b8a8 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py @@ -5,9 +5,8 @@ from models_library.generated_models.docker_rest_api import ContainerState from models_library.services_enums import ServiceState -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DockerContainerInspect, -) + +from ...models.dynamic_services_scheduler import DockerContainerInspect logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py index e6278a69083..bb2e9b92fb1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_abc.py @@ -3,7 +3,8 @@ from abc import ABC, abstractmethod from fastapi import FastAPI -from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData + +from .....models.dynamic_services_scheduler import SchedulerData class DynamicSchedulerEvent(ABC): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index 8d44b51d459..74dcfc72812 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -24,19 +24,19 @@ from servicelib.json_serialization import json_dumps from servicelib.rabbitmq import RabbitMQClient from simcore_postgres_database.models.comp_tasks import NodeClass -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DockerContainerInspect, - DockerStatus, - DynamicSidecarStatus, - NetworkId, - SchedulerData, -) from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed from .....core.settings import DynamicSidecarProxySettings, DynamicSidecarSettings +from .....models.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + DynamicSidecarStatus, + NetworkId, + SchedulerData, +) from .....utils.db import get_repository from .....utils.dict_utils import nested_update from ....catalog import CatalogClient diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index e56f136942f..24d31b07805 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -22,11 +22,6 @@ from servicelib.rabbitmq import RabbitMQClient from servicelib.utils import logged_gather from simcore_postgres_database.models.comp_tasks import NodeClass -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DockerContainerInspect, - DockerStatus, - SchedulerData, -) from tenacity import TryAgain from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -35,6 +30,11 @@ from .....core.errors import NodeRightsAcquireError from .....core.settings import AppSettings, DynamicSidecarSettings +from .....models.dynamic_services_scheduler import ( + DockerContainerInspect, + DockerStatus, + SchedulerData, +) from .....utils.db import get_repository from ....db.repositories.projects import ProjectsRepository from ....db.repositories.projects_networks import ProjectsNetworksRepository diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index b5ace09f00f..d9712d788bd 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -7,17 +7,17 @@ from fastapi import FastAPI from servicelib.error_codes import create_error_code -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DynamicSidecarStatus, - SchedulerData, - ServiceName, -) from .....core.settings import ( DynamicServicesSchedulerSettings, DynamicServicesSettings, DynamicSidecarSettings, ) +from .....models.dynamic_services_scheduler import ( + DynamicSidecarStatus, + SchedulerData, + ServiceName, +) from ...docker_api import ( are_sidecar_and_proxy_services_present, is_dynamic_sidecar_stack_missing, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index c6b5b5d436e..457831b12fc 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -36,12 +36,9 @@ from servicelib.background_task import cancel_task from servicelib.fastapi.long_running_tasks.client import ProgressCallback from servicelib.fastapi.long_running_tasks.server import TaskProgress -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - SchedulerData, - ServiceName, -) from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings +from .....models.dynamic_services_scheduler import SchedulerData, ServiceName from ...api_client import SidecarsClient, get_sidecars_client from ...docker_api import update_scheduler_data_label from ...errors import DynamicSidecarError, DynamicSidecarNotFoundError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index aede853010d..b7606404dc4 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -9,12 +9,9 @@ from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceBootType, ServiceState from servicelib.fastapi.long_running_tasks.client import ProgressCallback -from simcore_service_director_v2.models.dynamic_services_scheduler import ( - DynamicSidecarStatus, - SchedulerData, -) from .....core.settings import DynamicServicesSchedulerSettings, DynamicSidecarSettings +from .....models.dynamic_services_scheduler import DynamicSidecarStatus, SchedulerData from ...api_client import SidecarsClient, get_sidecars_client from ...docker_api import ( get_dynamic_sidecar_state, From 7d9b6b851042165c965c1bb1c649b34f2c038dac Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:31:24 +0200 Subject: [PATCH 16/20] relative import --- .../api_schemas_directorv2/comp_tasks.py | 11 ++++++----- .../models_library/api_schemas_directorv2/services.py | 3 ++- .../models-library/src/models_library/services.py | 6 ------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py index 146817229c4..ef840825b75 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py @@ -1,10 +1,11 @@ -from models_library.clusters import ClusterID -from models_library.projects import ProjectID -from models_library.projects_nodes import NodeID -from models_library.projects_pipeline import ComputationTask -from models_library.users import UserID from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, validator +from ..clusters import ClusterID +from ..projects import ProjectID +from ..projects_nodes_io import NodeID +from ..projects_pipeline import ComputationTask +from ..users import UserID + class ComputationGet(ComputationTask): url: AnyHttpUrl = Field( diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/services.py b/packages/models-library/src/models_library/api_schemas_directorv2/services.py index c8b68afd920..6e429be4d50 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/services.py @@ -1,9 +1,10 @@ from typing import Any, ClassVar -from models_library.service_settings_labels import ContainerSpec from pydantic import BaseModel, Field, validator from pydantic.types import ByteSize, NonNegativeInt +from ..service_settings_labels import ContainerSpec + class ServiceBuildDetails(BaseModel): build_date: str diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index c6c5535ca6a..500f184101b 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -1,9 +1,3 @@ -""" - -NOTE: to dump json-schema from CLI use - python -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > services-schema.json -""" - import re from datetime import datetime from enum import Enum From 9d0ea3e3f5c1183fa48e1b4bdc719051a0b93f11 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:39:20 +0200 Subject: [PATCH 17/20] mypy issues --- .../models/dynamic_services_scheduler.py | 6 +++--- .../modules/comp_scheduler/dask_scheduler.py | 2 +- .../modules/dynamic_sidecar/scheduler/_core/_scheduler.py | 6 +++++- .../dynamic_sidecar/scheduler/_core/_scheduler_utils.py | 2 +- .../src/simcore_service_director_v2/utils/dask.py | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index cdf06f67cae..0aa6a12cf41 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -467,19 +467,19 @@ def from_http_request( } if run_id: obj_dict["run_id"] = run_id - return cls.parse_obj(obj_dict) + return cls.parse_obj(obj_dict) # type: ignore[no-any-return] @classmethod def from_service_inspect( cls, service_inspect: Mapping[str, Any] ) -> "SchedulerData": labels = service_inspect["Spec"]["Labels"] - return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) + return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) # type: ignore[no-any-return] def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.copy( + return self.copy( # type: ignore[no-any-return] update={"compose_spec": json.dumps(self.compose_spec)}, deep=True ).json() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py index 11e9a0d4b54..c5c881885ff 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py @@ -27,6 +27,7 @@ from ...core.errors import TaskSchedulingError from ...core.settings import ComputationalBackendSettings +from ...models.comp_runs import MetadataDict from ...models.comp_tasks import CompTaskAtDB, Image from ...modules.dask_client import DaskClient from ...modules.dask_clients_pool import DaskClientsPool @@ -38,7 +39,6 @@ ) from ...utils.dask_client_utils import TaskHandlers from ..db.repositories.comp_tasks import CompTasksRepository -from ..models.comp_runs import MetadataDict from .base_scheduler import BaseCompScheduler logger = logging.getLogger(__name__) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 457831b12fc..5f2a983a072 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -325,7 +325,11 @@ async def mark_service_for_removal( ) return - current.dynamic_sidecar.service_removal_state.mark_to_remove(can_save) + # PC-> ANE: could you please review what to do when can_save=None + assert can_save is not None # nosec + current.dynamic_sidecar.service_removal_state.mark_to_remove( + can_save=can_save + ) await update_scheduler_data_label(current) # cancel current observation task diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index b7606404dc4..6b06a3a12a3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -104,7 +104,7 @@ async def discover_running_services(scheduler: "Scheduler") -> None: # type: ig def _create_model_from_scheduler_data( node_uuid: NodeID, - scheduler_data: "SchedulerData", # type: ignore + scheduler_data: SchedulerData, service_state: ServiceState, service_message: str, ) -> RunningDynamicServiceDetails: diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 61a4db58b67..1886c859890 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -459,7 +459,7 @@ def from_node_reqs_to_dask_resources( node_reqs: NodeRequirements, ) -> dict[str, int | float]: """Dask resources are set such as {"CPU": X.X, "GPU": Y.Y, "RAM": INT}""" - dask_resources = node_reqs.dict( + dask_resources: dict[str, int | float] = node_reqs.dict( exclude_unset=True, by_alias=True, exclude_none=True, From 89b7b3f67c18763995e82f7d147b9f8018bea976 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:50:35 +0200 Subject: [PATCH 18/20] minor catalog naming --- .../src/simcore_service_catalog/db/repositories/services.py | 2 +- .../{service_specifications.py => services_specifications.py} | 0 .../unit/with_dbs/test_api_routes_services_specifications.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename services/catalog/src/simcore_service_catalog/models/{service_specifications.py => services_specifications.py} (100%) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index 05f1d3cda72..70d60fbf991 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -22,7 +22,7 @@ from sqlalchemy.sql.expression import tuple_ from sqlalchemy.sql.selectable import Select -from ...models.service_specifications import ServiceSpecificationsAtDB +from ...models.services_specifications import ServiceSpecificationsAtDB from ..tables import services_access_rights, services_meta_data, services_specifications from ._base import BaseRepository diff --git a/services/catalog/src/simcore_service_catalog/models/service_specifications.py b/services/catalog/src/simcore_service_catalog/models/services_specifications.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/models/service_specifications.py rename to services/catalog/src/simcore_service_catalog/models/services_specifications.py diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py b/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py index 3b7051d2a54..40d4132e417 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py +++ b/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py @@ -34,7 +34,7 @@ from simcore_postgres_database.models.services_specifications import ( services_specifications, ) -from simcore_service_catalog.models.service_specifications import ( +from simcore_service_catalog.models.services_specifications import ( ServiceSpecificationsAtDB, ) from sqlalchemy.ext.asyncio import AsyncEngine From 3d4249c014a924f585c5f0004071870299a9eecb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:03:09 +0200 Subject: [PATCH 19/20] fixes improts --- services/director-v2/tests/unit/conftest.py | 10 +++++----- .../tests/unit/test_models_dynamic_services.py | 9 ++++----- .../test_models_schemas_dynamic_services_scheduler.py | 2 +- .../unit/test_modules_dynamic_sidecar_observer.py | 2 +- .../unit/test_modules_dynamic_sidecar_scheduler.py | 8 ++++---- ...ynamic_sidecar_scheduler__core__scheduler_utils.py | 2 +- ...t_modules_dynamic_sidecar_scheduler_core_events.py | 11 +++++++---- .../unit/with_dbs/test_api_route_dynamic_services.py | 6 +++--- services/director-v2/tests/unit/with_dbs/test_cli.py | 6 +++--- .../test_modules_dynamic_sidecar_docker_api.py | 6 ++---- ...st_modules_dynamic_sidecar_docker_service_specs.py | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index e5023988a6e..0f03835a093 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -28,6 +28,9 @@ from faker import Faker from fastapi import FastAPI from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate +from models_library.api_schemas_directorv2.dynamic_services_service import ( + ServiceDetails, +) from models_library.basic_types import PortInt from models_library.clusters import ClusterID from models_library.generated_models.docker_rest_api import ( @@ -35,6 +38,7 @@ ) from models_library.service_settings_labels import SimcoreServiceLabels from models_library.services import RunID, ServiceKey, ServiceKeyVersion, ServiceVersion +from models_library.services_enums import ServiceState from pydantic import parse_obj_as from pytest import LogCaptureFixture, MonkeyPatch from pytest_mock.plugin import MockerFixture @@ -43,11 +47,7 @@ from simcore_sdk.node_ports_v2 import FileLinkType from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.schemas.dynamic_services import ( - SchedulerData, - ServiceDetails, - ServiceState, -) +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs.volume_remover import ( DIND_VERSION, DockerVersion, diff --git a/services/director-v2/tests/unit/test_models_dynamic_services.py b/services/director-v2/tests/unit/test_models_dynamic_services.py index 214d83adeea..94c405b48e1 100644 --- a/services/director-v2/tests/unit/test_models_dynamic_services.py +++ b/services/director-v2/tests/unit/test_models_dynamic_services.py @@ -3,14 +3,13 @@ from collections import namedtuple import pytest +from models_library.api_schemas_directorv2.dynamic_services import ( + RunningDynamicServiceDetails, +) +from models_library.services_enums import ServiceBootType, ServiceState from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, -) -from simcore_service_director_v2.models.schemas.dynamic_services import ( - RunningDynamicServiceDetails, SchedulerData, - ServiceBootType, - ServiceState, ) from simcore_service_director_v2.modules.dynamic_sidecar.docker_states import ( CONTAINER_STATUSES_UNEXPECTED, diff --git a/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py b/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py index a16449560fd..607b8231f78 100644 --- a/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py +++ b/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py @@ -6,7 +6,7 @@ import pytest from pydantic import parse_file_as -from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData @pytest.fixture diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py index b2c99b61aed..c827ab11910 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py @@ -10,7 +10,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.api_client import ( setup, shutdown, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 9871ac2938a..c27ecdba858 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -14,19 +14,19 @@ import respx from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_directorv2.dynamic_services_service import ( + RunningDynamicServiceDetails, +) from models_library.service_settings_labels import SimcoreServiceLabels +from models_library.services_enums import ServiceState from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, -) -from simcore_service_director_v2.models.schemas.dynamic_services import ( DynamicSidecarStatus, - RunningDynamicServiceDetails, SchedulerData, - ServiceState, ) from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarError, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py index b0a00a366af..093ce666426 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py @@ -2,7 +2,7 @@ import pytest from pytest import LogCaptureFixture -from simcore_service_director_v2.models.schemas.dynamic_services import ( +from simcore_service_director_v2.models.dynamic_services_scheduler import ( DynamicSidecarStatus, SchedulerData, ) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py index ec9ef582423..453895d32b1 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_core_events.py @@ -3,7 +3,8 @@ import asyncio import logging -from typing import Final, Iterable +from collections.abc import Iterable +from typing import Final import pytest from fastapi import FastAPI @@ -16,8 +17,8 @@ ContainerState, DockerContainerInspect, DockerStatus, + SchedulerData, ) -from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.api_client import ( BaseClientHTTPError, ) @@ -57,7 +58,8 @@ def mock_sidecars_client_always_fail(mocker: MockerFixture) -> None: class MockedObj: @classmethod async def containers_inspect(cls, *args, **kwargs) -> None: - raise BaseClientHTTPError("will always fail") + msg = "will always fail" + raise BaseClientHTTPError(msg) mocker.patch.object(_events, "get_sidecars_client", return_value=MockedObj()) @@ -71,7 +73,8 @@ def __init__(self) -> None: async def containers_inspect(self, *args, **kwargs) -> None: self.counter += 1 if self.counter < STEPS / 2: - raise BaseClientHTTPError("will always fail") + msg = "will always fail" + raise BaseClientHTTPError(msg) mocker.patch.object(_events, "get_sidecars_client", return_value=MockedObj()) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index 9f257d37784..a23c2ee18c8 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -19,6 +19,9 @@ DynamicServiceCreate, RetrieveDataOutEnveloped, ) +from models_library.api_schemas_directorv2.dynamic_services_service import ( + RunningDynamicServiceDetails, +) from models_library.projects_nodes_io import NodeID from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture @@ -30,9 +33,6 @@ X_SIMCORE_USER_AGENT, ) from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData -from simcore_service_director_v2.models.schemas.dynamic_services import ( - RunningDynamicServiceDetails, -) from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarNotFoundError, ) diff --git a/services/director-v2/tests/unit/with_dbs/test_cli.py b/services/director-v2/tests/unit/with_dbs/test_cli.py index 55420ab64d1..4df7f8f65eb 100644 --- a/services/director-v2/tests/unit/with_dbs/test_cli.py +++ b/services/director-v2/tests/unit/with_dbs/test_cli.py @@ -14,6 +14,9 @@ from faker import Faker from fastapi import status from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_directorv2.dynamic_services_service import ( + RunningDynamicServiceDetails, +) from models_library.projects import ProjectAtDB from models_library.projects_nodes_io import NodeID from pytest_mock.plugin import MockerFixture @@ -23,9 +26,6 @@ from simcore_service_director_v2.cli._close_and_save_service import ( ThinDV2LocalhostClient, ) -from simcore_service_director_v2.models.schemas.dynamic_services import ( - RunningDynamicServiceDetails, -) from typer.testing import CliRunner pytest_simcore_core_services_selection = [ diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index 462a60d1589..e069c2308ab 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -19,6 +19,7 @@ from fastapi.encoders import jsonable_encoder from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID +from models_library.services_enums import ServiceState from models_library.users import UserID from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_director_v2.constants import ( @@ -30,11 +31,8 @@ from simcore_service_director_v2.core.settings import DynamicSidecarSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, - SimcoreServiceLabels, -) -from simcore_service_director_v2.models.schemas.dynamic_services import ( SchedulerData, - ServiceState, + SimcoreServiceLabels, ) from simcore_service_director_v2.modules.dynamic_sidecar import docker_api from simcore_service_director_v2.modules.dynamic_sidecar.docker_api._core import ( diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 6f639cdd384..c41a92fe1e7 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -22,7 +22,7 @@ from pytest_simcore.helpers.utils_envs import setenvs_from_dict from servicelib.json_serialization import json_dumps from simcore_service_director_v2.core.settings import DynamicSidecarSettings -from simcore_service_director_v2.models.schemas.dynamic_services import SchedulerData +from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.catalog import CatalogClient from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs import ( get_dynamic_sidecar_spec, From 2e535710cb8bbca94a24654f8471dfd87b187297 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:07:58 +0200 Subject: [PATCH 20/20] missing func --- .../scheduler/_core/_scheduler_utils.py | 14 +++++++------- .../tests/unit/test_models_dynamic_services.py | 10 +++++++++- ...dynamic_sidecar_docker_service_specs_sidecar.py | 1 + .../unit/test_modules_dynamic_sidecar_scheduler.py | 11 +++++++---- ...mic_sidecar_scheduler__core__scheduler_utils.py | 4 ++++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index 6b06a3a12a3..ea513502c0a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -102,7 +102,7 @@ async def discover_running_services(scheduler: "Scheduler") -> None: # type: ig await scheduler._add_service(scheduler_data) # pylint: disable=protected-access -def _create_model_from_scheduler_data( +def create_model_from_scheduler_data( node_uuid: NodeID, scheduler_data: SchedulerData, service_state: ServiceState, @@ -132,7 +132,7 @@ async def get_stack_status_from_scheduler_data( # check if there was an error picked up by the scheduler # and marked this service as failed if scheduler_data.dynamic_sidecar.status.current != DynamicSidecarStatus.OK: - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.FAILED, @@ -141,7 +141,7 @@ async def get_stack_status_from_scheduler_data( # is the service stopping? if scheduler_data.dynamic_sidecar.service_removal_state.can_remove: - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.STOPPING, @@ -157,7 +157,7 @@ async def get_stack_status_from_scheduler_data( ) except DockerServiceNotFoundError: # in this case, the service is starting, so state is pending - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.PENDING, @@ -166,7 +166,7 @@ async def get_stack_status_from_scheduler_data( # while the dynamic-sidecar state is not RUNNING report it's state if sidecar_state != ServiceState.RUNNING: - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=sidecar_state, @@ -179,7 +179,7 @@ async def get_stack_status_from_scheduler_data( # wait for containers to start if len(scheduler_data.dynamic_sidecar.containers_inspect) == 0: # marks status as waiting for containers - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.STARTING, @@ -190,7 +190,7 @@ async def get_stack_status_from_scheduler_data( container_state, container_message = extract_containers_minimum_statuses( scheduler_data.dynamic_sidecar.containers_inspect ) - return _create_model_from_scheduler_data( + return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=container_state, diff --git a/services/director-v2/tests/unit/test_models_dynamic_services.py b/services/director-v2/tests/unit/test_models_dynamic_services.py index 94c405b48e1..dd0df8a0eed 100644 --- a/services/director-v2/tests/unit/test_models_dynamic_services.py +++ b/services/director-v2/tests/unit/test_models_dynamic_services.py @@ -1,4 +1,9 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import string from collections import namedtuple @@ -15,6 +20,9 @@ CONTAINER_STATUSES_UNEXPECTED, extract_containers_minimum_statuses, ) +from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._scheduler_utils import ( + create_model_from_scheduler_data, +) # the following is the predefined expected ordering, change below test only if # this order is not adequate anymore @@ -121,7 +129,7 @@ def _all_states() -> set[ServiceState]: def test_running_service_details_make_status( scheduler_data: SchedulerData, service_message: str, service_state: ServiceState ): - running_service_details = RunningDynamicServiceDetails.from_scheduler_data( + running_service_details = create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=service_state, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 41726be30b7..016908bdc21 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -4,6 +4,7 @@ from typing import Any +import pytest from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs.sidecar import ( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index c27ecdba858..99b68d80a10 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -45,6 +45,9 @@ from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._scheduler import ( Scheduler, ) +from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._scheduler_utils import ( + create_model_from_scheduler_data, +) # running scheduler at a hight rate to stress out the system # and ensure faster tests @@ -364,7 +367,7 @@ async def test_get_stack_status( await scheduler._scheduler._add_service(scheduler_data) stack_status = await scheduler.get_stack_status(scheduler_data.node_uuid) - assert stack_status == RunningDynamicServiceDetails.from_scheduler_data( + assert stack_status == create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.PENDING, @@ -395,7 +398,7 @@ async def test_get_stack_status_failing_sidecar( await scheduler._scheduler._add_service(scheduler_data) stack_status = await scheduler.get_stack_status(scheduler_data.node_uuid) - assert stack_status == RunningDynamicServiceDetails.from_scheduler_data( + assert stack_status == create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.FAILED, @@ -414,7 +417,7 @@ async def test_get_stack_status_containers_are_starting( async with _assert_get_dynamic_services_mocked( scheduler, scheduler_data, mock_service_running, expected_status="created" ) as stack_status: - assert stack_status == RunningDynamicServiceDetails.from_scheduler_data( + assert stack_status == create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.STARTING, @@ -433,7 +436,7 @@ async def test_get_stack_status_ok( async with _assert_get_dynamic_services_mocked( scheduler, scheduler_data, mock_service_running, expected_status="running" ) as stack_status: - assert stack_status == RunningDynamicServiceDetails.from_scheduler_data( + assert stack_status == create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, service_state=ServiceState.RUNNING, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py index 093ce666426..f60dae9a80e 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler__core__scheduler_utils.py @@ -1,4 +1,8 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable import pytest from pytest import LogCaptureFixture