improving tests

Author: sanderegg

packages/models-library/src/models_library/clusters.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import Any, ClassVar, Final, Literal, TypeAlias
 
+from attr import frozen
 from pydantic import (
     AnyUrl,
     BaseModel,
@@ -46,6 +47,7 @@ class BaseAuthentication(BaseModel):
     type: str
 
     class Config:
+        frozen = True
         extra = Extra.forbid
 
 

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -1,5 +1,8 @@
 version: "3.8"
 x-dask-tls-secrets: &dask_tls_secrets
+  - source: dask_tls_ca
+    target: ${DASK_TLS_KEY}
+    mode: 0444
   - source: dask_tls_key
     target: ${DASK_TLS_KEY}
     mode: 0444
@@ -48,7 +51,7 @@ services:
       DASK_TLS_CERT: ${DASK_TLS_CERT}
       DASK_TLS_KEY: ${DASK_TLS_KEY}
       LOG_LEVEL: ${LOG_LEVEL}
-      SIDECAR_COMP_SERVICES_SHARED_FOLDER: ${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data}
+      SIDECAR_COMP_SERVICES_SHARED_FOLDER: /home/scu/computational_shared_data
       SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: computational_shared_data
     deploy:
       mode: global
@@ -115,7 +118,9 @@ volumes:
   redis-data:
 
 secrets:
+  dask_tls_ca:
+    file: .dask-certificates/tls_dask_ca.pem
   dask_tls_key:
-    file: ./dask-sidecar/.dask-certificates/dask-key.pem
+    file: .dask-certificates/tls_dask_cert.pem
   dask_tls_cert:
-    file: ./dask-sidecar/.dask-certificates/dask-cert.pem
+    file: .dask-certificates/tls_dask_key.pem

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py

@@ -2,6 +2,7 @@
 import datetime
 import functools
 import json
+from pathlib import Path
 from typing import Any, Final
 
 from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData, EC2Tags
@@ -10,7 +11,7 @@
     ClusterState,
     OnDemandCluster,
 )
-from models_library.clusters import NoAuthentication
+from models_library.clusters import NoAuthentication, TLSAuthentication
 from models_library.users import UserID
 from models_library.wallets import WalletID
 from types_aiobotocore_ec2.literals import InstanceStateNameType
@@ -20,14 +21,35 @@
 from .dask import get_scheduler_url
 
 _DOCKER_COMPOSE_FILE_NAME: Final[str] = "docker-compose.yml"
+_HOST_DOCKER_COMPOSE_PATH: Final[Path] = Path(f"/{_DOCKER_COMPOSE_FILE_NAME}")
+_HOST_CERTIFICATES_BASE_PATH: Final[Path] = Path("/.dask-sidecar-certificates")
+_HOST_TLS_CA_FILE_PATH: Final[Path] = _HOST_CERTIFICATES_BASE_PATH / "tls_dask_ca.pem"
+_HOST_TLS_CERT_FILE_PATH: Final[Path] = (
+    _HOST_CERTIFICATES_BASE_PATH / "tls_dask_cert.pem"
+)
+_HOST_TLS_KEY_FILE_PATH: Final[Path] = _HOST_CERTIFICATES_BASE_PATH / "tls_dask_key.pem"
+
+
+def _base_64_encode(file: Path) -> str:
+    assert file.exists()  # nosec
+    with file.open("rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
 
 
 @functools.lru_cache
 def _docker_compose_yml_base64_encoded() -> str:
     file_path = PACKAGE_DATA_FOLDER / _DOCKER_COMPOSE_FILE_NAME
-    assert file_path.exists()  # nosec
-    with file_path.open("rb") as f:
-        return base64.b64encode(f.read()).decode("utf-8")
+    return _base_64_encode(file_path)
+
+
+@functools.lru_cache
+def _write_tls_certificates_commands(auth: TLSAuthentication) -> list[str]:
+    return [
+        f"mkdir --parents {_HOST_CERTIFICATES_BASE_PATH}",
+        f"echo '{_base_64_encode(auth.tls_ca_file)}' > {_HOST_TLS_CA_FILE_PATH}",
+        f"echo '{_base_64_encode(auth.tls_client_cert)}' > {_HOST_TLS_CERT_FILE_PATH}",
+        f"echo '{_base_64_encode(auth.tls_client_key)}' > {_HOST_TLS_KEY_FILE_PATH}",
+    ]
 
 
 def _prepare_environment_variables(
@@ -47,21 +69,24 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         return f"'{json.dumps(jsonable_encoder(entries))}'"
 
     return [
-        f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}",
-        f"DASK_NTHREADS={app_settings.CLUSTERS_KEEPER_DASK_NTHREADS or ''}",
         f"CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}",
         f"CLUSTERS_KEEPER_EC2_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}",
         f"CLUSTERS_KEEPER_EC2_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}",
         f"CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}",
+        f"DASK_NTHREADS={app_settings.CLUSTERS_KEEPER_DASK_NTHREADS or ''}",
+        f"DASK_TLS_CA_FILE={_HOST_TLS_CA_FILE_PATH}",
+        f"DASK_TLS_CERT={_HOST_TLS_CERT_FILE_PATH}",
+        f"DASK_TLS_KEY={_HOST_TLS_KEY_FILE_PATH}",
+        f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}",
+        f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}",
+        f"LOG_LEVEL={app_settings.LOG_LEVEL}",
         f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}",
+        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}",  # type: ignore
         f"WORKERS_EC2_INSTANCES_KEY_NAME={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME}",
         f"WORKERS_EC2_INSTANCES_MAX_INSTANCES={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_MAX_INSTANCES}",
-        f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}",
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
-        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}",  # type: ignore
-        f"LOG_LEVEL={app_settings.LOG_LEVEL}",
     ]
 
 
@@ -82,13 +107,23 @@ def create_startup_script(
     )
 
     startup_commands = ec2_boot_specific.custom_boot_scripts.copy()
+
+    if isinstance(
+        app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH,
+        TLSAuthentication,
+    ):
+        write_certificates_commands = _write_tls_certificates_commands(
+            app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH
+        )
+        startup_commands.extend(write_certificates_commands)
+
     startup_commands.extend(
         [
             # NOTE: https://stackoverflow.com/questions/41203492/solving-redis-warnings-on-overcommit-memory-and-transparent-huge-pages-for-ubunt
             "sysctl vm.overcommit_memory=1",
-            f"echo '{_docker_compose_yml_base64_encoded()}' | base64 -d > docker-compose.yml",
+            f"echo '{_docker_compose_yml_base64_encoded()}' | base64 -d > {_HOST_DOCKER_COMPOSE_PATH}",
             "docker swarm init",
-            f"{' '.join(environment_variables)} docker stack deploy --with-registry-auth --compose-file=docker-compose.yml dask_stack",
+            f"{' '.join(environment_variables)} docker stack deploy --with-registry-auth --compose-file={_HOST_DOCKER_COMPOSE_PATH} dask_stack",
         ]
     )
     return "\n".join(startup_commands)

services/clusters-keeper/tests/unit/test_utils_clusters.py

@@ -67,12 +67,12 @@ def test_create_startup_script(
     for boot_script in ec2_boot_specs.custom_boot_scripts:
         assert boot_script in startup_script
     # we have commands to pipe into a docker-compose file
-    assert " | base64 -d > docker-compose.yml" in startup_script
+    assert " | base64 -d > /docker-compose.yml" in startup_script
     # we have commands to init a docker-swarm
     assert "docker swarm init" in startup_script
     # we have commands to deploy a stack
     assert (
-        "docker stack deploy --with-registry-auth --compose-file=docker-compose.yml dask_stack"
+        "docker stack deploy --with-registry-auth --compose-file=/docker-compose.yml dask_stack"
         in startup_script
     )
     # before that we have commands that setup ENV variables, let's check we have all of them as defined in the docker-compose
@@ -87,20 +87,20 @@ def test_create_startup_script(
     )
     startup_script_env_keys_names = [key for key, _ in startup_script_key_value_pairs]
     # docker-compose expected values
+    docker_compose_expected_environment: dict[str, str] = {}
     assert "services" in clusters_keeper_docker_compose
-    assert "autoscaling" in clusters_keeper_docker_compose["services"]
-    assert "environment" in clusters_keeper_docker_compose["services"]["autoscaling"]
-    docker_compose_expected_environment: dict[
-        str, str
-    ] = clusters_keeper_docker_compose["services"]["autoscaling"]["environment"]
-    assert isinstance(docker_compose_expected_environment, dict)
+    assert isinstance(clusters_keeper_docker_compose["services"], dict)
+    for service_details in clusters_keeper_docker_compose["services"].values():
+        if "environment" in service_details:
+            assert isinstance(service_details["environment"], dict)
+            docker_compose_expected_environment |= service_details["environment"]
 
     # check the expected environment variables are set so the docker-compose will be complete (we define enough)
     expected_env_keys = [
         v[2:-1].split(":")[0]
         for v in docker_compose_expected_environment.values()
         if isinstance(v, str) and v.startswith("${")
-    ] + ["DASK_NTHREADS", "DOCKER_IMAGE_TAG"]
+    ] + ["DOCKER_IMAGE_TAG"]
     for env_key in expected_env_keys:
         assert (
             env_key in startup_script_env_keys_names
@@ -173,7 +173,6 @@ def test_startup_script_defines_all_envs_for_docker_compose(
     _ENV_VARIABLE_NOT_SET_ERROR = "variable is not set"
     assert _ENV_VARIABLE_NOT_SET_ERROR not in process.stderr.decode()
     assert process.stdout
-    assert process.stdout is None
 
 
 @pytest.mark.parametrize(