From 7088cbe58ec8538ac24b7efe59b21c7fdb868b36 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 5 Nov 2025 16:10:32 +0100 Subject: [PATCH 01/11] feat: indicate supported platforms when checking a session image --- Makefile | 4 + .../notebooks/oci/__init__.py | 1 + .../notebooks/oci/base_model.py | 15 ++ .../notebooks/oci/manifest_list.py | 156 ++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 components/renku_data_services/notebooks/oci/__init__.py create mode 100644 components/renku_data_services/notebooks/oci/base_model.py create mode 100644 components/renku_data_services/notebooks/oci/manifest_list.py diff --git a/Makefile b/Makefile index 218648a2c..10c39fd40 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,10 @@ amalthea_schema: ## Updates generates pydantic classes from CRDs shipwright_schema: ## Updates the Shipwright pydantic classes curl https://raw.githubusercontent.com/shipwright-io/build/refs/tags/v0.15.2/deploy/crds/shipwright.io_buildruns.yaml | yq '.spec.versions[] | select(.name == "v1beta1") | .schema.openAPIV3Schema' | poetry run datamodel-codegen --output components/renku_data_services/session/cr_shipwright_buildrun.py --base-class renku_data_services.session.cr_base.BaseCRD ${CR_CODEGEN_PARAMS} +.PHONY: oci_schema +oci_schema: ## Updates the OCI classes + poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json" --output components/renku_data_services/notebooks/oci/manifest_list.py --class-name ManifestList --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS} + ##@ Devcontainer .PHONY: devcontainer_up diff --git a/components/renku_data_services/notebooks/oci/__init__.py b/components/renku_data_services/notebooks/oci/__init__.py new file mode 100644 index 000000000..b2c6d3fb7 --- /dev/null +++ b/components/renku_data_services/notebooks/oci/__init__.py @@ -0,0 +1 @@ +"""Classes and utilities about OCI images.""" diff --git a/components/renku_data_services/notebooks/oci/base_model.py b/components/renku_data_services/notebooks/oci/base_model.py new file mode 100644 index 000000000..5f4177385 --- /dev/null +++ b/components/renku_data_services/notebooks/oci/base_model.py @@ -0,0 +1,15 @@ +"""Base model for generated classes.""" + +from pydantic import BaseModel + + +class BaseOciModel(BaseModel): + """Base API specification.""" + + class Config: + """Enables orm mode for pydantic.""" + + from_attributes = True + # NOTE: By default the pydantic library does not use python for regex but a rust crate + # this rust crate does not support lookahead regex syntax but we need it in this component + regex_engine = "python-re" diff --git a/components/renku_data_services/notebooks/oci/manifest_list.py b/components/renku_data_services/notebooks/oci/manifest_list.py new file mode 100644 index 000000000..ddf02b65c --- /dev/null +++ b/components/renku_data_services/notebooks/oci/manifest_list.py @@ -0,0 +1,156 @@ +# generated by datamodel-codegen: +# filename: https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json +# timestamp: 2025-11-05T15:09:20+00:00 + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any, Optional + +from pydantic import AnyUrl, ConfigDict, Field, RootModel + +from renku_data_services.notebooks.oci.base_model import BaseOciModel + + +class Platform(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + architecture: str + os: str + os_version: Optional[str] = Field(default=None, alias="os.version") + os_features: Optional[Sequence[str]] = Field(default=None, alias="os.features") + variant: Optional[str] = None + + +class Int8(RootModel[int]): + root: int = Field(..., ge=-128, le=127) + + +class Int16(RootModel[int]): + root: int = Field(..., ge=-32768, le=32767) + + +class Int32(RootModel[int]): + root: int = Field(..., ge=-2147483648, le=2147483647) + + +class Uint8(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class Uint16(RootModel[int]): + root: int = Field(..., ge=0, le=65535) + + +class Uint32(RootModel[int]): + root: int = Field(..., ge=0, le=4294967295) + + +class Uint64(RootModel[int]): + root: int = Field(..., ge=0, le=18446744073709552000) + + +class Uint16Pointer(RootModel[Optional[Uint16]]): + root: Optional[Uint16] + + +class Uint64Pointer(RootModel[Optional[Uint64]]): + root: Optional[Uint64] + + +class StringPointer(RootModel[Optional[str]]): + root: Optional[str] + + +class MapStringObject(RootModel[Mapping[str, Mapping[str, Any]]]): + root: Mapping[str, Mapping[str, Any]] + + +class ContentDescriptor(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + mediaType: str = Field( + ..., + description="the mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + size: int = Field( + ..., + description="the size in bytes of the referenced object", + ge=-9223372036854776000, + le=9223372036854776000, + ) + digest: str = Field( + ..., + description="the cryptographic checksum digest of the object, in the pattern ':'", + pattern="^[a-z0-9]+(?:[+._-][a-z0-9]+)*:[a-zA-Z0-9=_-]+$", + ) + urls: Optional[Sequence[AnyUrl]] = Field( + default=None, + description="a list of urls from which this object may be downloaded", + ) + data: Optional[str] = Field( + default=None, + description="an embedding of the targeted content (base64 encoded)", + ) + artifactType: Optional[str] = Field( + default=None, + description="the IANA media type of this artifact", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + annotations: Optional[Mapping[str, str]] = None + + +class Manifest(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + mediaType: str = Field( + ..., + description="the mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + size: int = Field( + ..., + description="the size in bytes of the referenced object", + ge=-9223372036854776000, + le=9223372036854776000, + ) + digest: str = Field( + ..., + description="the cryptographic checksum digest of the object, in the pattern ':'", + pattern="^[a-z0-9]+(?:[+._-][a-z0-9]+)*:[a-zA-Z0-9=_-]+$", + ) + urls: Optional[Sequence[AnyUrl]] = Field( + default=None, + description="a list of urls from which this object may be downloaded", + ) + platform: Optional[Platform] = None + annotations: Optional[Mapping[str, str]] = None + + +class ManifestList(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + schemaVersion: int = Field( + ..., + description="This field specifies the image index schema version as an integer", + ge=2, + le=2, + ) + mediaType: Optional[str] = Field( + default=None, + description="the mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + artifactType: Optional[str] = Field( + default=None, + description="the artifact mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + subject: Optional[ContentDescriptor] = None + manifests: Sequence[Manifest] + annotations: Optional[Mapping[str, str]] = None From ed6e5639c78455fc7253c611db98d3b98e44aefa Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 11:25:29 +0100 Subject: [PATCH 02/11] wip --- Makefile | 4 +- bases/renku_data_services/data_api/app.py | 16 +-- .../data_api/dependencies.py | 7 + .../notebooks/api/classes/image.py | 20 ++- .../notebooks/blueprints.py | 29 ++--- .../notebooks/core_sessions.py | 23 ++-- .../notebooks/image_check.py | 123 ++++++++---------- .../notebooks/oci/image_config.py | 117 +++++++++++++++++ .../oci/{manifest_list.py => image_index.py} | 8 +- .../notebooks/oci/image_manifest.py | 116 +++++++++++++++++ .../notebooks/oci/models.py | 12 ++ .../notebooks/oci/utils.py | 54 ++++++++ pyproject.toml | 3 + 13 files changed, 420 insertions(+), 112 deletions(-) create mode 100644 components/renku_data_services/notebooks/oci/image_config.py rename components/renku_data_services/notebooks/oci/{manifest_list.py => image_index.py} (96%) create mode 100644 components/renku_data_services/notebooks/oci/image_manifest.py create mode 100644 components/renku_data_services/notebooks/oci/models.py create mode 100644 components/renku_data_services/notebooks/oci/utils.py diff --git a/Makefile b/Makefile index 10c39fd40..46238cf14 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,9 @@ shipwright_schema: ## Updates the Shipwright pydantic classes .PHONY: oci_schema oci_schema: ## Updates the OCI classes - poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json" --output components/renku_data_services/notebooks/oci/manifest_list.py --class-name ManifestList --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS} + poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/26647a49f642c7d22a1cd3aa0a48e4650a542269/schema/config-schema.json" --output components/renku_data_services/notebooks/oci/image_config.py --class-name ImageConfig --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS} + poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json" --output components/renku_data_services/notebooks/oci/image_index.py --class-name ImageIndex --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS} + poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-manifest-schema.json" --output components/renku_data_services/notebooks/oci/image_manifest.py --class-name ImageManifest --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS} ##@ Devcontainer diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index be6e4ba83..b72dbe481 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -205,19 +205,19 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: url_prefix=url_prefix, authenticator=dm.authenticator, nb_config=dm.config.nb_config, + cluster_repo=dm.cluster_repo, + data_connector_repo=dm.data_connector_repo, + data_connector_secret_repo=dm.data_connector_secret_repo, + git_provider_helper=dm.git_provider_helper, + image_check_repo=dm.image_check_repo, + internal_gitlab_authenticator=dm.gitlab_authenticator, + metrics=dm.metrics, project_repo=dm.project_repo, project_session_secret_repo=dm.project_session_secret_repo, + rp_repo=dm.rp_repo, session_repo=dm.session_repo, storage_repo=dm.storage_repo, - rp_repo=dm.rp_repo, user_repo=dm.kc_user_repo, - data_connector_repo=dm.data_connector_repo, - data_connector_secret_repo=dm.data_connector_secret_repo, - cluster_repo=dm.cluster_repo, - internal_gitlab_authenticator=dm.gitlab_authenticator, - metrics=dm.metrics, - connected_svcs_repo=dm.connected_services_repo, - git_provider_helper=dm.git_provider_helper, ) platform_config = PlatformConfigBP( name="platform_config", diff --git a/bases/renku_data_services/data_api/dependencies.py b/bases/renku_data_services/data_api/dependencies.py index 2df6d993a..3132cb618 100644 --- a/bases/renku_data_services/data_api/dependencies.py +++ b/bases/renku_data_services/data_api/dependencies.py @@ -54,6 +54,7 @@ from renku_data_services.notebooks.api.classes.data_service import DummyGitProviderHelper, GitProviderHelper from renku_data_services.notebooks.config import GitProviderHelperProto, get_clusters from renku_data_services.notebooks.constants import AMALTHEA_SESSION_GVK, JUPYTER_SESSION_GVK +from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository from renku_data_services.project.db import ( ProjectMemberRepository, @@ -138,6 +139,7 @@ class DependencyManager: data_connector_repo: DataConnectorRepository data_connector_secret_repo: DataConnectorSecretRepository cluster_repo: ClusterRepository + image_check_repo: ImageCheckRepository metrics_repo: MetricsRepository metrics: StagingMetricsService shipwright_client: ShipwrightClient | None @@ -372,6 +374,10 @@ def from_env(cls) -> DependencyManager: secret_service_public_key=config.secrets.public_key, authz=authz, ) + image_check_repo = ImageCheckRepository( + nb_config=config.nb_config, + connected_services_repo=connected_services_repo, + ) search_reprovisioning = SearchReprovision( search_updates_repo=search_updates_repo, reprovisioning_repo=reprovisioning_repo, @@ -410,6 +416,7 @@ def from_env(cls) -> DependencyManager: data_connector_repo=data_connector_repo, data_connector_secret_repo=data_connector_secret_repo, cluster_repo=cluster_repo, + image_check_repo=image_check_repo, metrics_repo=metrics_repo, metrics=metrics, shipwright_client=shipwright_client, diff --git a/components/renku_data_services/notebooks/api/classes/image.py b/components/renku_data_services/notebooks/api/classes/image.py index a8170a6a0..1c90438b2 100644 --- a/components/renku_data_services/notebooks/api/classes/image.py +++ b/components/renku_data_services/notebooks/api/classes/image.py @@ -142,22 +142,32 @@ def platform_matches(manifest: dict[str, Any]) -> bool: async def image_exists(self, image: Image) -> bool: """Check the docker repo API if the image exists.""" - return await self.image_check(image) == 200 + status_code, _ = await self.image_check(image) + return status_code == 200 - async def image_check(self, image: Image) -> int: + async def image_check(self, image: Image, include_manifest: bool = False) -> tuple[int, httpx.Response]: """Check the image at the registry.""" token = await self._get_docker_token(image) image_digest_url = f"{self.scheme}://{image.hostname}/v2/{image.name}/manifests/{image.tag}" accept_media = ",".join( - [e.value for e in [ManifestTypes.docker_v2, ManifestTypes.oci_v1_manifest, ManifestTypes.oci_v1_index]] + [ + e.value + for e in [ + ManifestTypes.docker_v2, + ManifestTypes.docker_v2_list, + ManifestTypes.oci_v1_manifest, + ManifestTypes.oci_v1_index, + ] + ] ) headers = {"Accept": accept_media} if token: headers["Authorization"] = f"Bearer {token}" + method = "GET" if include_manifest else "HEAD" - res = await self.client.head(image_digest_url, headers=headers) + res = await self.client.request(method, image_digest_url, headers=headers) logger.debug(f"Checked image access: {image_digest_url}: {res.status_code}") - return res.status_code + return res.status_code, res async def get_image_config(self, image: Image) -> Optional[dict[str, Any]]: """Query the docker API to get the configuration of an image.""" diff --git a/components/renku_data_services/notebooks/blueprints.py b/components/renku_data_services/notebooks/blueprints.py index 329fe56b6..bc2533bed 100644 --- a/components/renku_data_services/notebooks/blueprints.py +++ b/components/renku_data_services/notebooks/blueprints.py @@ -12,7 +12,6 @@ from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint from renku_data_services.base_models import AnonymousAPIUser, APIUser, AuthenticatedAPIUser, Authenticator from renku_data_services.base_models.metrics import MetricsService -from renku_data_services.connected_services.db import ConnectedServicesRepository from renku_data_services.connected_services.models import ConnectionStatus from renku_data_services.crc.db import ClusterRepository, ResourcePoolRepository from renku_data_services.data_connectors.db import ( @@ -20,7 +19,7 @@ DataConnectorSecretRepository, ) from renku_data_services.errors import errors -from renku_data_services.notebooks import apispec, core, image_check +from renku_data_services.notebooks import apispec, core from renku_data_services.notebooks.api.classes.image import Image from renku_data_services.notebooks.api.schemas.config_server_options import ServerOptionsEndpointResponse from renku_data_services.notebooks.api.schemas.logs import ServerLogs @@ -31,6 +30,7 @@ validate_session_post_request, ) from renku_data_services.notebooks.errors.intermittent import AnonymousUserPatchError +from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.project.db import ProjectRepository, ProjectSessionSecretRepository from renku_data_services.session.db import SessionRepository from renku_data_services.storage.db import StorageRepository @@ -196,18 +196,18 @@ class NotebooksNewBP(CustomBlueprint): authenticator: base_models.Authenticator internal_gitlab_authenticator: base_models.Authenticator nb_config: NotebooksConfig + cluster_repo: ClusterRepository + data_connector_repo: DataConnectorRepository + data_connector_secret_repo: DataConnectorSecretRepository + git_provider_helper: GitProviderHelperProto + image_check_repo: ImageCheckRepository project_repo: ProjectRepository project_session_secret_repo: ProjectSessionSecretRepository - session_repo: SessionRepository rp_repo: ResourcePoolRepository + session_repo: SessionRepository storage_repo: StorageRepository user_repo: UserRepo - data_connector_repo: DataConnectorRepository - data_connector_secret_repo: DataConnectorSecretRepository metrics: MetricsService - cluster_repo: ClusterRepository - connected_svcs_repo: ConnectedServicesRepository - git_provider_helper: GitProviderHelperProto def start(self) -> BlueprintFactoryResponse: """Start a session with the new operator.""" @@ -236,7 +236,7 @@ async def _handler( session_repo=self.session_repo, user_repo=self.user_repo, metrics=self.metrics, - connected_svcs_repo=self.connected_svcs_repo, + image_check_repo=self.image_check_repo, ) status = 201 if created else 200 return json(session.as_apispec().model_dump(exclude_none=True, mode="json"), status) @@ -303,7 +303,7 @@ async def _handler( rp_repo=self.rp_repo, session_repo=self.session_repo, metrics=self.metrics, - connected_svcs_repo=self.connected_svcs_repo, + image_check_repo=self.image_check_repo, ) return json(new_session.as_apispec().model_dump(exclude_none=True, mode="json")) @@ -337,11 +337,10 @@ async def _check_docker_image( query: apispec.SessionsImagesGetParametersQuery, ) -> JSONResponse: image = Image.from_path(query.image_url) - result = await image_check.check_image( - image, - user, - self.connected_svcs_repo, - image_check.InternalGitLabConfig(internal_gitlab_user, self.nb_config), + result = await self.image_check_repo.check_image( + user=user, + gitlab_user=internal_gitlab_user, + image=image, ) logger.info(f"Checked image {query.image_url}: {result}") conn = None diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index 078adec84..8acb36fe2 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -18,11 +18,9 @@ from ulid import ULID from yaml import safe_dump -import renku_data_services.notebooks.image_check as ic from renku_data_services.app_config import logging from renku_data_services.base_models import AnonymousAPIUser, APIUser, AuthenticatedAPIUser from renku_data_services.base_models.metrics import MetricsService -from renku_data_services.connected_services.db import ConnectedServicesRepository from renku_data_services.crc.db import ClusterRepository, ResourcePoolRepository from renku_data_services.crc.models import ( ClusterSettings, @@ -79,6 +77,7 @@ State, Storage, ) +from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.notebooks.models import ( ExtraSecret, SessionDataConnectorOverride, @@ -563,11 +562,11 @@ def __format_image_pull_secret(secret_name: str, access_token: str, registry_dom async def __get_connected_services_image_pull_secret( - secret_name: str, connected_svcs_repo: ConnectedServicesRepository, image: str, user: APIUser + secret_name: str, image_check_repo: ImageCheckRepository, image: str, user: APIUser ) -> ExtraSecret | None: """Return a secret for accessing the image if one is available for the given user.""" image_parsed = Image.from_path(image) - image_check_result = await ic.check_image(image_parsed, user, connected_svcs_repo, None) + image_check_result = await image_check_repo.check_image(user=user, gitlab_user=None, image=image_parsed) logger.debug(f"Set pull secret for {image} to connection {image_check_result.image_provider}") if not image_check_result.token: return None @@ -588,12 +587,12 @@ async def get_image_pull_secret( nb_config: NotebooksConfig, user: APIUser, internal_gitlab_user: APIUser, - connected_svcs_repo: ConnectedServicesRepository, + image_check_repo: ImageCheckRepository, ) -> ExtraSecret | None: """Get an image pull secret.""" v2_secret = await __get_connected_services_image_pull_secret( - f"{server_name}-image-secret", connected_svcs_repo, image, user + f"{server_name}-image-secret", image_check_repo, image, user ) if v2_secret: return v2_secret @@ -672,7 +671,8 @@ async def start_session( session_repo: SessionRepository, user_repo: UserRepo, metrics: MetricsService, - connected_svcs_repo: ConnectedServicesRepository, + # connected_svcs_repo: ConnectedServicesRepository, + image_check_repo: ImageCheckRepository, ) -> tuple[AmaltheaSessionV1Alpha1, bool]: """Start an Amalthea session. @@ -840,7 +840,8 @@ async def start_session( nb_config=nb_config, user=user, internal_gitlab_user=internal_gitlab_user, - connected_svcs_repo=connected_svcs_repo, + # connected_svcs_repo=connected_svcs_repo, + image_check_repo=image_check_repo, ) if image_secret: session_extras = session_extras.concat(SessionExtraResources(secrets=[image_secret])) @@ -983,7 +984,8 @@ async def patch_session( project_session_secret_repo: ProjectSessionSecretRepository, rp_repo: ResourcePoolRepository, session_repo: SessionRepository, - connected_svcs_repo: ConnectedServicesRepository, + # connected_svcs_repo: ConnectedServicesRepository, + image_check_repo: ImageCheckRepository, metrics: MetricsService, ) -> AmaltheaSessionV1Alpha1: """Patch an Amalthea session.""" @@ -1119,7 +1121,8 @@ async def patch_session( image=image, server_name=server_name, nb_config=nb_config, - connected_svcs_repo=connected_svcs_repo, + # connected_svcs_repo=connected_svcs_repo, + image_check_repo=image_check_repo, user=user, internal_gitlab_user=internal_gitlab_user, ) diff --git a/components/renku_data_services/notebooks/image_check.py b/components/renku_data_services/notebooks/image_check.py index 88207227e..aca4ff297 100644 --- a/components/renku_data_services/notebooks/image_check.py +++ b/components/renku_data_services/notebooks/image_check.py @@ -27,6 +27,7 @@ from renku_data_services.errors import errors from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI from renku_data_services.notebooks.config import NotebooksConfig +from renku_data_services.notebooks.oci.image_index import Platform logger = logging.getLogger(__name__) @@ -37,6 +38,7 @@ class CheckResult: """Result of checking access to an image.""" accessible: bool + platforms: list[Platform] | None response_code: int image_provider: ImageProvider | None = None token: str | None = field(default=None, repr=False) @@ -77,73 +79,58 @@ def user(self) -> APIUser | None: return self.image_provider.connected_user.user -@dataclass -class InternalGitLabConfig: - """Required for internal gitlab, which will be shut down soon.""" +class ImageCheckRepository: + """Repository for checking session images with rich responses.""" + + def __init__(self, nb_config: NotebooksConfig, connected_services_repo: ConnectedServicesRepository) -> None: + self.nb_config = nb_config + self.connected_services_repo = connected_services_repo + + async def check_image(self, user: APIUser, gitlab_user: APIUser | None, image: Image) -> CheckResult: + """Check access to the given image and provide image and access details.""" + reg_api: ImageRepoDockerAPI = image.repo_api() # public images + unauth_error: errors.UnauthorizedError | None = None + image_provider = await self.connected_services_repo.get_provider_for_image(user, image) + connected_user = image_provider.connected_user if image_provider is not None else None + connection = connected_user.connection if connected_user is not None else None + if image_provider is not None: + try: + reg_api = await self.connected_services_repo.get_image_repo_client(image_provider) + except errors.UnauthorizedError as e: + logger.info(f"Error getting image repo client for image {image}: {e}") + unauth_error = e + except OAuthError as e: + logger.info(f"Error getting image repo client for image {image}: {e}") + unauth_error = errors.UnauthorizedError( + message=f"OAuth error when getting repo client for image: {image}" + ) + unauth_error.__cause__ = e + elif gitlab_user and gitlab_user.access_token and image.hostname == self.nb_config.git.registry: + logger.debug(f"Using internal gitlab at {self.nb_config.git.registry}") + reg_api = reg_api.with_oauth2_token(gitlab_user.access_token) - gitlab_user: APIUser - nb_config: NotebooksConfig - - -async def check_image_path( - image_path: str, - user: APIUser, - connected_services: ConnectedServicesRepository, - internal_gitlab_config: InternalGitLabConfig | None, -) -> CheckResult: - """Check access to the given image.""" - image = Image.from_path(image_path) - return await check_image(image, user, connected_services, internal_gitlab_config) - - -async def check_image( - image: Image, - user: APIUser, - connected_services: ConnectedServicesRepository, - intern_gl_cfg: InternalGitLabConfig | None, -) -> CheckResult: - """Check access to the given image.""" - - reg_api: ImageRepoDockerAPI = image.repo_api() # public images - unauth_error: errors.UnauthorizedError | None = None - image_provider = await connected_services.get_provider_for_image(user, image) - connected_user = image_provider.connected_user if image_provider is not None else None - connection = connected_user.connection if connected_user is not None else None - if image_provider is not None: - try: - reg_api = await connected_services.get_image_repo_client(image_provider) - except errors.UnauthorizedError as e: - logger.info(f"Error getting image repo client for image {image}: {e}") - unauth_error = e - except OAuthError as e: - logger.info(f"Error getting image repo client for image {image}: {e}") - unauth_error = errors.UnauthorizedError(message=f"OAuth error when getting repo client for image: {image}") - unauth_error.__cause__ = e - elif ( - intern_gl_cfg - and image.hostname == intern_gl_cfg.nb_config.git.registry - and intern_gl_cfg.gitlab_user.access_token - ): - logger.debug(f"Using internal gitlab at {intern_gl_cfg.nb_config.git.registry}") - reg_api = reg_api.with_oauth2_token(intern_gl_cfg.gitlab_user.access_token) - - try: - result = await reg_api.image_check(image) - except httpx.HTTPError as e: - logger.info(f"Error connecting {reg_api.scheme}://{reg_api.hostname}: {e}") - result = 0 - - if result != 200 and connection is not None: try: - await connected_services.get_oauth2_connected_account(connection.id, user) - except errors.UnauthorizedError as e: - logger.info(f"Error getting connected account: {e}") - unauth_error = e - - return CheckResult( - accessible=result == 200, - response_code=result, - image_provider=image_provider, - token=reg_api.oauth2_token, - error=unauth_error, - ) + status_code, response = await reg_api.image_check(image, include_manifest=True) + except httpx.HTTPError as e: + logger.info(f"Error connecting {reg_api.scheme}://{reg_api.hostname}: {e}") + status_code = 0 + + if status_code != 200 and connection is not None: + try: + await self.connected_services_repo.get_oauth2_connected_account(connection.id, user) + except errors.UnauthorizedError as e: + logger.info(f"Error getting connected account: {e}") + unauth_error = e + + if status_code == 200: + # Get platforms information + pass + + return CheckResult( + accessible=status_code == 200, + platforms=None, + response_code=status_code, + image_provider=image_provider, + token=reg_api.oauth2_token, + error=unauth_error, + ) diff --git a/components/renku_data_services/notebooks/oci/image_config.py b/components/renku_data_services/notebooks/oci/image_config.py new file mode 100644 index 000000000..ad0dec2a9 --- /dev/null +++ b/components/renku_data_services/notebooks/oci/image_config.py @@ -0,0 +1,117 @@ +# generated by datamodel-codegen: +# filename: https://raw.githubusercontent.com/opencontainers/image-spec/26647a49f642c7d22a1cd3aa0a48e4650a542269/schema/config-schema.json +# timestamp: 2025-11-06T08:57:15+00:00 + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import datetime +from enum import Enum +from typing import Any, Optional + +from pydantic import ConfigDict, Field, RootModel + +from renku_data_services.notebooks.oci.base_model import BaseOciModel + + +class Type(Enum): + layers = "layers" + + +class Rootfs(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + diff_ids: Sequence[str] + type: Type + + +class HistoryItem(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + created: Optional[datetime] = None + author: Optional[str] = None + created_by: Optional[str] = None + comment: Optional[str] = None + empty_layer: Optional[bool] = None + + +class Int8(RootModel[int]): + root: int = Field(..., ge=-128, le=127) + + +class Int16(RootModel[int]): + root: int = Field(..., ge=-32768, le=32767) + + +class Int32(RootModel[int]): + root: int = Field(..., ge=-2147483648, le=2147483647) + + +class Int64(RootModel[int]): + root: int = Field(..., ge=-9223372036854776000, le=9223372036854776000) + + +class Uint8(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class Uint16(RootModel[int]): + root: int = Field(..., ge=0, le=65535) + + +class Uint32(RootModel[int]): + root: int = Field(..., ge=0, le=4294967295) + + +class Uint64(RootModel[int]): + root: int = Field(..., ge=0, le=18446744073709552000) + + +class Uint16Pointer(RootModel[Optional[Uint16]]): + root: Optional[Uint16] + + +class Uint64Pointer(RootModel[Optional[Uint64]]): + root: Optional[Uint64] + + +class Base64(RootModel[str]): + root: str + + +class StringPointer(RootModel[Optional[str]]): + root: Optional[str] + + +class Config(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + User: Optional[str] = None + ExposedPorts: Optional[Mapping[str, Mapping[str, Any]]] = None + Env: Optional[Sequence[str]] = None + Entrypoint: Optional[Sequence[str]] = None + Cmd: Optional[Sequence[str]] = None + Volumes: Optional[Mapping[str, Mapping[str, Any]]] = None + WorkingDir: Optional[str] = None + Labels: Optional[Mapping[str, str]] = None + StopSignal: Optional[str] = None + ArgsEscaped: Optional[bool] = None + + +class ImageConfig(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + created: Optional[datetime] = None + author: Optional[str] = None + architecture: str + variant: Optional[str] = None + os: str + os_version: Optional[str] = Field(default=None, alias="os.version") + os_features: Optional[Sequence[str]] = Field(default=None, alias="os.features") + config: Optional[Config] = None + rootfs: Rootfs + history: Optional[Sequence[HistoryItem]] = None diff --git a/components/renku_data_services/notebooks/oci/manifest_list.py b/components/renku_data_services/notebooks/oci/image_index.py similarity index 96% rename from components/renku_data_services/notebooks/oci/manifest_list.py rename to components/renku_data_services/notebooks/oci/image_index.py index ddf02b65c..57b7dc920 100644 --- a/components/renku_data_services/notebooks/oci/manifest_list.py +++ b/components/renku_data_services/notebooks/oci/image_index.py @@ -1,14 +1,12 @@ # generated by datamodel-codegen: # filename: https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json -# timestamp: 2025-11-05T15:09:20+00:00 +# timestamp: 2025-11-06T08:57:17+00:00 from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Any, Optional +from typing import Any, Mapping, Optional, Sequence from pydantic import AnyUrl, ConfigDict, Field, RootModel - from renku_data_services.notebooks.oci.base_model import BaseOciModel @@ -131,7 +129,7 @@ class Manifest(BaseOciModel): annotations: Optional[Mapping[str, str]] = None -class ManifestList(BaseOciModel): +class ImageIndex(BaseOciModel): model_config = ConfigDict( extra="allow", ) diff --git a/components/renku_data_services/notebooks/oci/image_manifest.py b/components/renku_data_services/notebooks/oci/image_manifest.py new file mode 100644 index 000000000..6a629355c --- /dev/null +++ b/components/renku_data_services/notebooks/oci/image_manifest.py @@ -0,0 +1,116 @@ +# generated by datamodel-codegen: +# filename: https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-manifest-schema.json +# timestamp: 2025-11-06T08:57:18+00:00 + +from __future__ import annotations + +from typing import Any, Mapping, Optional, Sequence + +from pydantic import AnyUrl, ConfigDict, Field, RootModel +from renku_data_services.notebooks.oci.base_model import BaseOciModel + + +class Int8(RootModel[int]): + root: int = Field(..., ge=-128, le=127) + + +class Int16(RootModel[int]): + root: int = Field(..., ge=-32768, le=32767) + + +class Int32(RootModel[int]): + root: int = Field(..., ge=-2147483648, le=2147483647) + + +class Uint8(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class Uint16(RootModel[int]): + root: int = Field(..., ge=0, le=65535) + + +class Uint32(RootModel[int]): + root: int = Field(..., ge=0, le=4294967295) + + +class Uint64(RootModel[int]): + root: int = Field(..., ge=0, le=18446744073709552000) + + +class Uint16Pointer(RootModel[Optional[Uint16]]): + root: Optional[Uint16] + + +class Uint64Pointer(RootModel[Optional[Uint64]]): + root: Optional[Uint64] + + +class StringPointer(RootModel[Optional[str]]): + root: Optional[str] + + +class MapStringObject(RootModel[Mapping[str, Mapping[str, Any]]]): + root: Mapping[str, Mapping[str, Any]] + + +class ContentDescriptor(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + mediaType: str = Field( + ..., + description="the mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + size: int = Field( + ..., + description="the size in bytes of the referenced object", + ge=-9223372036854776000, + le=9223372036854776000, + ) + digest: str = Field( + ..., + description="the cryptographic checksum digest of the object, in the pattern ':'", + pattern="^[a-z0-9]+(?:[+._-][a-z0-9]+)*:[a-zA-Z0-9=_-]+$", + ) + urls: Optional[Sequence[AnyUrl]] = Field( + default=None, + description="a list of urls from which this object may be downloaded", + ) + data: Optional[str] = Field( + default=None, + description="an embedding of the targeted content (base64 encoded)", + ) + artifactType: Optional[str] = Field( + default=None, + description="the IANA media type of this artifact", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + annotations: Optional[Mapping[str, str]] = None + + +class ImageManifest(BaseOciModel): + model_config = ConfigDict( + extra="allow", + ) + schemaVersion: int = Field( + ..., + description="This field specifies the image manifest schema version as an integer", + ge=2, + le=2, + ) + mediaType: Optional[str] = Field( + default=None, + description="the mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + artifactType: Optional[str] = Field( + default=None, + description="the artifact mediatype of the referenced object", + pattern="^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$", + ) + config: ContentDescriptor + subject: Optional[ContentDescriptor] = None + layers: Sequence[ContentDescriptor] = Field(..., min_length=1) + annotations: Optional[Mapping[str, str]] = None diff --git a/components/renku_data_services/notebooks/oci/models.py b/components/renku_data_services/notebooks/oci/models.py new file mode 100644 index 000000000..90ba90424 --- /dev/null +++ b/components/renku_data_services/notebooks/oci/models.py @@ -0,0 +1,12 @@ +"""Constants related to OCI images.""" + +from enum import StrEnum + + +class ManifestMediaTypes(StrEnum): + """The content types related to OCI image manifests.""" + + docker_manifest_v2 = "application/vnd.docker.distribution.manifest.v2+json" + docker_list_v2 = "application/vnd.docker.distribution.manifest.list.v2+json" + oci_manifest_v1 = "application/vnd.oci.image.manifest.v1+json" + oci_index_v1 = "application/vnd.oci.image.index.v1+json" diff --git a/components/renku_data_services/notebooks/oci/utils.py b/components/renku_data_services/notebooks/oci/utils.py new file mode 100644 index 000000000..c5ba6aebd --- /dev/null +++ b/components/renku_data_services/notebooks/oci/utils.py @@ -0,0 +1,54 @@ +"""Utilities for OCI images.""" + + +import httpx + +from renku_data_services import errors +from renku_data_services.app_config import logging +from renku_data_services.notebooks.oci.image_index import ImageIndex, Platform +from renku_data_services.notebooks.oci.image_manifest import ImageManifest +from renku_data_services.notebooks.oci.models import ManifestMediaTypes + +logger = logging.getLogger(__name__) + + +def get_image_platforms(manifest_response: httpx.Response) -> list[Platform] | None: + """Returns the list of platforms supported by the image manifest.""" + try: + parsed = parse_manifest_response(manifest_response) + except Exception as err: + logger.warning(f"Error parsing image manifest: {err}") + return None + + if isinstance(parsed, ImageIndex): + platforms: list[Platform] = [] + for manifest in parsed.manifests: + # Ignore manifests without a platform + if ( + manifest.platform is None + or manifest.platform.os == "unknown" + or manifest.platform.architecture == "unknown" + ): + continue + platforms.append(manifest.platform) + return platforms + + # parsed.config + + +def parse_manifest_response(response: httpx.Response) -> ImageIndex | ImageManifest: + """Parse a manifest response.""" + + content_type = response.headers.get("Content-Type") + if content_type not in { + ManifestMediaTypes.docker_list_v2, + ManifestMediaTypes.docker_manifest_v2, + ManifestMediaTypes.oci_index_v1, + ManifestMediaTypes.oci_manifest_v1, + }: + raise errors.ValidationError(message=f"Unexpected content type {content_type}.") + + if content_type in {ManifestMediaTypes.docker_list_v2, ManifestMediaTypes.oci_index_v1}: + return ImageIndex.model_validate_json(response.content) + else: + return ImageManifest.model_validate_json(response.content) diff --git a/pyproject.toml b/pyproject.toml index d2e069a8c..20883f2b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,9 @@ exclude = [ "components/renku_data_services/notebooks/cr_amalthea_session.py", "components/renku_data_services/notebooks/cr_jupyter_server.py", "components/renku_data_services/session/cr_shipwright_buildrun.py", + "components/renku_data_services/notebooks/oci/image_config.py", + "components/renku_data_services/notebooks/oci/image_index.py", + "components/renku_data_services/notebooks/oci/image_manifest.py", ] [tool.ruff.format] From 0eecea1c825a205e59b11ee08b4422fbc9564044 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 10:51:38 +0000 Subject: [PATCH 03/11] wip --- .../notebooks/api/classes/image.py | 20 +++++---- .../notebooks/image_check.py | 11 +++-- .../notebooks/oci/models.py | 2 + .../notebooks/oci/utils.py | 45 ++++++++++++++++++- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/components/renku_data_services/notebooks/api/classes/image.py b/components/renku_data_services/notebooks/api/classes/image.py index 1c90438b2..a2ac0cd0c 100644 --- a/components/renku_data_services/notebooks/api/classes/image.py +++ b/components/renku_data_services/notebooks/api/classes/image.py @@ -169,6 +169,17 @@ async def image_check(self, image: Image, include_manifest: bool = False) -> tup logger.debug(f"Checked image access: {image_digest_url}: {res.status_code}") return res.status_code, res + async def get_image_config_from_digest(self, image: Image, config_digest: str) -> httpx.Response: + """Query the docker API to get the configuration of an image from the config digest.""" + token = await self._get_docker_token(image) + return await self.client.get( + f"{self.scheme}://{image.hostname}/v2/{image.name}/blobs/{config_digest}", + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {token}", + }, + ) + async def get_image_config(self, image: Image) -> Optional[dict[str, Any]]: """Query the docker API to get the configuration of an image.""" manifest = await self.get_image_manifest(image) @@ -177,14 +188,7 @@ async def get_image_config(self, image: Image) -> Optional[dict[str, Any]]: config_digest = manifest.get("config", {}).get("digest") if config_digest is None: return None - token = await self._get_docker_token(image) - res = await self.client.get( - f"{self.scheme}://{image.hostname}/v2/{image.name}/blobs/{config_digest}", - headers={ - "Accept": "application/json", - "Authorization": f"Bearer {token}", - }, - ) + res = await self.get_image_config_from_digest(image, config_digest) if res.status_code != 200: return None return cast(dict[str, Any], res.json()) diff --git a/components/renku_data_services/notebooks/image_check.py b/components/renku_data_services/notebooks/image_check.py index aca4ff297..2d42770d3 100644 --- a/components/renku_data_services/notebooks/image_check.py +++ b/components/renku_data_services/notebooks/image_check.py @@ -28,6 +28,7 @@ from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.notebooks.oci.image_index import Platform +from renku_data_services.notebooks.oci.utils import get_image_platforms logger = logging.getLogger(__name__) @@ -114,6 +115,7 @@ async def check_image(self, user: APIUser, gitlab_user: APIUser | None, image: I except httpx.HTTPError as e: logger.info(f"Error connecting {reg_api.scheme}://{reg_api.hostname}: {e}") status_code = 0 + response = None if status_code != 200 and connection is not None: try: @@ -122,13 +124,14 @@ async def check_image(self, user: APIUser, gitlab_user: APIUser | None, image: I logger.info(f"Error getting connected account: {e}") unauth_error = e - if status_code == 200: - # Get platforms information - pass + platforms = None + if status_code == 200 and response is not None: + platforms = await get_image_platforms(manifest_response=response, image=image, reg_api=reg_api) + logger.info(f"Platforms: {platforms}") return CheckResult( accessible=status_code == 200, - platforms=None, + platforms=platforms, response_code=status_code, image_provider=image_provider, token=reg_api.oauth2_token, diff --git a/components/renku_data_services/notebooks/oci/models.py b/components/renku_data_services/notebooks/oci/models.py index 90ba90424..ec49e2bbe 100644 --- a/components/renku_data_services/notebooks/oci/models.py +++ b/components/renku_data_services/notebooks/oci/models.py @@ -6,7 +6,9 @@ class ManifestMediaTypes(StrEnum): """The content types related to OCI image manifests.""" + docker_config_v2 = "application/vnd.docker.container.image.v1+json" docker_manifest_v2 = "application/vnd.docker.distribution.manifest.v2+json" docker_list_v2 = "application/vnd.docker.distribution.manifest.list.v2+json" + oci_config_v1 = "application/vnd.oci.image.config.v1+json" oci_manifest_v1 = "application/vnd.oci.image.manifest.v1+json" oci_index_v1 = "application/vnd.oci.image.index.v1+json" diff --git a/components/renku_data_services/notebooks/oci/utils.py b/components/renku_data_services/notebooks/oci/utils.py index c5ba6aebd..a932c841d 100644 --- a/components/renku_data_services/notebooks/oci/utils.py +++ b/components/renku_data_services/notebooks/oci/utils.py @@ -1,18 +1,25 @@ """Utilities for OCI images.""" +from typing import TYPE_CHECKING import httpx from renku_data_services import errors from renku_data_services.app_config import logging +from renku_data_services.notebooks.oci.image_config import ImageConfig from renku_data_services.notebooks.oci.image_index import ImageIndex, Platform from renku_data_services.notebooks.oci.image_manifest import ImageManifest from renku_data_services.notebooks.oci.models import ManifestMediaTypes +if TYPE_CHECKING: + from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI + logger = logging.getLogger(__name__) -def get_image_platforms(manifest_response: httpx.Response) -> list[Platform] | None: +async def get_image_platforms( + manifest_response: httpx.Response, image: "Image", reg_api: "ImageRepoDockerAPI" +) -> list[Platform] | None: """Returns the list of platforms supported by the image manifest.""" try: parsed = parse_manifest_response(manifest_response) @@ -33,7 +40,28 @@ def get_image_platforms(manifest_response: httpx.Response) -> list[Platform] | N platforms.append(manifest.platform) return platforms - # parsed.config + try: + config_response = await reg_api.get_image_config_from_digest(image=image, config_digest=parsed.config.digest) + except Exception as err: + logger.warning(f"Error getting image config: {err}") + return None + + try: + parsed_config = parse_config_response(config_response) + except Exception as err: + logger.warning(f"Error parsing image config: {err}") + return None + + platform = Platform.model_validate( + { + "architecture": parsed_config.architecture, + "os": parsed_config.os, + "os.feature": parsed_config.os_features, + "os.version": parsed_config.os_version, + "variant": parsed_config.variant, + }, + ) + return [platform] def parse_manifest_response(response: httpx.Response) -> ImageIndex | ImageManifest: @@ -52,3 +80,16 @@ def parse_manifest_response(response: httpx.Response) -> ImageIndex | ImageManif return ImageIndex.model_validate_json(response.content) else: return ImageManifest.model_validate_json(response.content) + + +def parse_config_response(response: httpx.Response) -> ImageConfig: + """Parse a config response.""" + + content_type = response.headers.get("Content-Type") + if content_type not in { + ManifestMediaTypes.docker_config_v2, + ManifestMediaTypes.oci_config_v1, + }: + raise errors.ValidationError(message=f"Unexpected content type {content_type}.") + + return ImageConfig.model_validate_json(response.content) From 886001156516a6b1098b23139d1eb9cabd66113b Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 12:02:55 +0000 Subject: [PATCH 04/11] fix content-type --- components/renku_data_services/notebooks/oci/utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/components/renku_data_services/notebooks/oci/utils.py b/components/renku_data_services/notebooks/oci/utils.py index a932c841d..b283cd33e 100644 --- a/components/renku_data_services/notebooks/oci/utils.py +++ b/components/renku_data_services/notebooks/oci/utils.py @@ -66,7 +66,6 @@ async def get_image_platforms( def parse_manifest_response(response: httpx.Response) -> ImageIndex | ImageManifest: """Parse a manifest response.""" - content_type = response.headers.get("Content-Type") if content_type not in { ManifestMediaTypes.docker_list_v2, @@ -84,12 +83,6 @@ def parse_manifest_response(response: httpx.Response) -> ImageIndex | ImageManif def parse_config_response(response: httpx.Response) -> ImageConfig: """Parse a config response.""" - - content_type = response.headers.get("Content-Type") - if content_type not in { - ManifestMediaTypes.docker_config_v2, - ManifestMediaTypes.oci_config_v1, - }: - raise errors.ValidationError(message=f"Unexpected content type {content_type}.") - + # NOTE: it seems that the "Content-Type" is set to "application/octet-stream" by the Docker registry + # NOTE: so we do not validate it return ImageConfig.model_validate_json(response.content) From 7a4aeb95970cbd1756a341c4cc5fbe7f41f0feba Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 12:53:46 +0000 Subject: [PATCH 05/11] return platforms --- .../notebooks/api.spec.yaml | 26 +++++++++++++++++ .../renku_data_services/notebooks/apispec.py | 13 ++++++++- .../notebooks/blueprints.py | 8 ++++- .../notebooks/image_check.py | 2 +- .../notebooks/oci/models.py | 12 ++++++++ .../notebooks/oci/utils.py | 29 ++++++++++++------- 6 files changed, 76 insertions(+), 14 deletions(-) diff --git a/components/renku_data_services/notebooks/api.spec.yaml b/components/renku_data_services/notebooks/api.spec.yaml index 10fd3cbc0..1f470cd83 100644 --- a/components/renku_data_services/notebooks/api.spec.yaml +++ b/components/renku_data_services/notebooks/api.spec.yaml @@ -1076,6 +1076,8 @@ components: accessible: type: boolean description: Whether the image is accessible or not. + platforms: + $ref: "#/components/schemas/ImagePlatforms" connection: "$ref": "#/components/schemas/ImageConnection" provider: @@ -1114,6 +1116,30 @@ components: - id - name - url + ImagePlatforms: + type: array + description: The list of platforms an image can run on. + items: + $ref: "#/components/schemas/ImagePlatform" + ImagePlatform: + type: object + description: A runtime platform. + properties: + architecture: + type: string + os: + type: string + os.version: + type: string + os.features: + type: array + items: + type: string + variant: + type: string + required: + - architecture + - os responses: ImageCheckResponse: description: Information about whether a docker image is available or not and if there is a connected service then which connected service can be used to access the image. diff --git a/components/renku_data_services/notebooks/apispec.py b/components/renku_data_services/notebooks/apispec.py index 67f6ab916..09e23d527 100644 --- a/components/renku_data_services/notebooks/apispec.py +++ b/components/renku_data_services/notebooks/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2025-10-15T12:41:50+00:00 +# timestamp: 2025-11-06T12:51:12+00:00 from __future__ import annotations @@ -279,6 +279,14 @@ class ImageProvider(BaseAPISpec): url: str +class ImagePlatform(BaseAPISpec): + architecture: str + os: str + os_version: Optional[str] = Field(None, alias="os.version") + os_features: Optional[List[str]] = Field(None, alias="os.features") + variant: Optional[str] = None + + class NotebooksImagesGetParametersQuery(BaseAPISpec): image_url: str = Field(..., min_length=1) @@ -448,6 +456,9 @@ class SessionListResponse(RootModel[List[SessionResponse]]): class ImageCheckResponse(BaseAPISpec): accessible: bool = Field(..., description="Whether the image is accessible or not.") + platforms: Optional[List[ImagePlatform]] = Field( + None, description="The list of platforms an image can run on." + ) connection: Optional[ImageConnection] = None provider: Optional[ImageProvider] = None diff --git a/components/renku_data_services/notebooks/blueprints.py b/components/renku_data_services/notebooks/blueprints.py index bc2533bed..a85e9986e 100644 --- a/components/renku_data_services/notebooks/blueprints.py +++ b/components/renku_data_services/notebooks/blueprints.py @@ -365,7 +365,13 @@ async def _check_docker_image( id=result.client.id, name=result.client.display_name, url=result.client.url ) - resp = apispec.ImageCheckResponse(accessible=result.accessible, connection=conn, provider=provider) + platforms = None + if result.platforms: + platforms = [apispec.ImagePlatform.model_validate(p) for p in result.platforms] + + resp = apispec.ImageCheckResponse( + accessible=result.accessible, platforms=platforms, connection=conn, provider=provider + ) return json(resp.model_dump(exclude_none=True, mode="json")) diff --git a/components/renku_data_services/notebooks/image_check.py b/components/renku_data_services/notebooks/image_check.py index 2d42770d3..020166663 100644 --- a/components/renku_data_services/notebooks/image_check.py +++ b/components/renku_data_services/notebooks/image_check.py @@ -27,7 +27,7 @@ from renku_data_services.errors import errors from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI from renku_data_services.notebooks.config import NotebooksConfig -from renku_data_services.notebooks.oci.image_index import Platform +from renku_data_services.notebooks.oci.models import Platform from renku_data_services.notebooks.oci.utils import get_image_platforms logger = logging.getLogger(__name__) diff --git a/components/renku_data_services/notebooks/oci/models.py b/components/renku_data_services/notebooks/oci/models.py index ec49e2bbe..6c5bf59f3 100644 --- a/components/renku_data_services/notebooks/oci/models.py +++ b/components/renku_data_services/notebooks/oci/models.py @@ -1,5 +1,6 @@ """Constants related to OCI images.""" +from dataclasses import dataclass from enum import StrEnum @@ -12,3 +13,14 @@ class ManifestMediaTypes(StrEnum): oci_config_v1 = "application/vnd.oci.image.config.v1+json" oci_manifest_v1 = "application/vnd.oci.image.manifest.v1+json" oci_index_v1 = "application/vnd.oci.image.index.v1+json" + + +@dataclass(frozen=True, eq=True, kw_only=True, order=True) +class Platform: + """Represents a runtime platform.""" + + architecture: str + os: str + variant: str | None = None + os_version: str | None = None + os_features: list[str] | None = None diff --git a/components/renku_data_services/notebooks/oci/utils.py b/components/renku_data_services/notebooks/oci/utils.py index b283cd33e..0ee6539c0 100644 --- a/components/renku_data_services/notebooks/oci/utils.py +++ b/components/renku_data_services/notebooks/oci/utils.py @@ -7,9 +7,9 @@ from renku_data_services import errors from renku_data_services.app_config import logging from renku_data_services.notebooks.oci.image_config import ImageConfig -from renku_data_services.notebooks.oci.image_index import ImageIndex, Platform +from renku_data_services.notebooks.oci.image_index import ImageIndex from renku_data_services.notebooks.oci.image_manifest import ImageManifest -from renku_data_services.notebooks.oci.models import ManifestMediaTypes +from renku_data_services.notebooks.oci.models import ManifestMediaTypes, Platform if TYPE_CHECKING: from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI @@ -37,7 +37,16 @@ async def get_image_platforms( or manifest.platform.architecture == "unknown" ): continue - platforms.append(manifest.platform) + platforms.append( + Platform( + architecture=manifest.platform.architecture, + os=manifest.platform.os, + os_features=list(manifest.platform.os_features) if manifest.platform.os_features else None, + os_version=manifest.platform.os_version, + variant=manifest.platform.variant, + ) + ) + platforms = sorted(set(platforms)) return platforms try: @@ -52,14 +61,12 @@ async def get_image_platforms( logger.warning(f"Error parsing image config: {err}") return None - platform = Platform.model_validate( - { - "architecture": parsed_config.architecture, - "os": parsed_config.os, - "os.feature": parsed_config.os_features, - "os.version": parsed_config.os_version, - "variant": parsed_config.variant, - }, + platform = Platform( + architecture=parsed_config.architecture, + os=parsed_config.os, + os_features=list(parsed_config.os_features) if parsed_config.os_features else None, + os_version=parsed_config.os_version, + variant=parsed_config.variant, ) return [platform] From 7eacceca4ed09508f237b47d60c3f77a844bf53e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 13:03:45 +0000 Subject: [PATCH 06/11] fix sort order --- components/renku_data_services/notebooks/oci/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/notebooks/oci/models.py b/components/renku_data_services/notebooks/oci/models.py index 6c5bf59f3..2347ba822 100644 --- a/components/renku_data_services/notebooks/oci/models.py +++ b/components/renku_data_services/notebooks/oci/models.py @@ -19,8 +19,8 @@ class ManifestMediaTypes(StrEnum): class Platform: """Represents a runtime platform.""" - architecture: str os: str + architecture: str variant: str | None = None os_version: str | None = None os_features: list[str] | None = None From 151de84d5dc0fc4f7cb203293539e6c43a5b5ef8 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 13:30:19 +0000 Subject: [PATCH 07/11] fix tests? --- test/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/utils.py b/test/utils.py index edc48ec15..b02894deb 100644 --- a/test/utils.py +++ b/test/utils.py @@ -31,6 +31,7 @@ from renku_data_services.metrics.db import MetricsRepository from renku_data_services.namespace.db import GroupRepository from renku_data_services.notebooks.api.classes.data_service import GitProviderHelper +from renku_data_services.notebooks.image_check import ImageCheckRepository from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository from renku_data_services.project.db import ( ProjectMemberRepository, @@ -298,6 +299,9 @@ def from_env( data_connector_repo=data_connector_repo, ) cluster_repo = ClusterRepository(session_maker=config.db.async_session_maker) + image_check_repo = ImageCheckRepository( + nb_config=config.nb_config, connected_services_repo=connected_services_repo + ) metrics_repo = MetricsRepository(session_maker=config.db.async_session_maker) git_provider_helper = GitProviderHelper(connected_services_repo, "", "", "", config.enable_internal_gitlab) return cls( @@ -329,6 +333,7 @@ def from_env( data_connector_repo=data_connector_repo, data_connector_secret_repo=data_connector_secret_repo, cluster_repo=cluster_repo, + image_check_repo=image_check_repo, metrics_repo=metrics_repo, metrics=metrics_mock, shipwright_client=None, From fc908091834a0215457bb51ac230249222fb9873 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 19:16:06 +0100 Subject: [PATCH 08/11] Apply suggestion from @leafty --- components/renku_data_services/notebooks/core_sessions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index 8acb36fe2..018d17496 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -671,7 +671,6 @@ async def start_session( session_repo: SessionRepository, user_repo: UserRepo, metrics: MetricsService, - # connected_svcs_repo: ConnectedServicesRepository, image_check_repo: ImageCheckRepository, ) -> tuple[AmaltheaSessionV1Alpha1, bool]: """Start an Amalthea session. From 51c857377ec031133275431c2bb841f2b76a6b53 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 19:16:27 +0100 Subject: [PATCH 09/11] Apply suggestion from @leafty --- components/renku_data_services/notebooks/core_sessions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index 018d17496..20fdef496 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -1120,7 +1120,6 @@ async def patch_session( image=image, server_name=server_name, nb_config=nb_config, - # connected_svcs_repo=connected_svcs_repo, image_check_repo=image_check_repo, user=user, internal_gitlab_user=internal_gitlab_user, From 989d259fe2a0d146994eb4b27d3d9ac20017ed02 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 19:16:47 +0100 Subject: [PATCH 10/11] Apply suggestion from @leafty --- components/renku_data_services/notebooks/core_sessions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index 20fdef496..ec7e33680 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -839,7 +839,6 @@ async def start_session( nb_config=nb_config, user=user, internal_gitlab_user=internal_gitlab_user, - # connected_svcs_repo=connected_svcs_repo, image_check_repo=image_check_repo, ) if image_secret: From 252d6884996c352e8f0bce28da920f8d0b869649 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 6 Nov 2025 19:17:24 +0100 Subject: [PATCH 11/11] Update components/renku_data_services/notebooks/core_sessions.py --- components/renku_data_services/notebooks/core_sessions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index ec7e33680..17f6e821e 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -982,7 +982,6 @@ async def patch_session( project_session_secret_repo: ProjectSessionSecretRepository, rp_repo: ResourcePoolRepository, session_repo: SessionRepository, - # connected_svcs_repo: ConnectedServicesRepository, image_check_repo: ImageCheckRepository, metrics: MetricsService, ) -> AmaltheaSessionV1Alpha1: