Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ amalthea_schema: ## Updates generates pydantic classes from CRDs
shipwright_schema: ## Updates the Shipwright pydantic classes
curl https://raw.githubusercontent.com/shipwright-io/build/refs/tags/v0.15.2/deploy/crds/shipwright.io_buildruns.yaml | yq '.spec.versions[] | select(.name == "v1beta1") | .schema.openAPIV3Schema' | poetry run datamodel-codegen --output components/renku_data_services/session/cr_shipwright_buildrun.py --base-class renku_data_services.session.cr_base.BaseCRD ${CR_CODEGEN_PARAMS}

.PHONY: oci_schema
oci_schema: ## Updates the OCI classes
poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/26647a49f642c7d22a1cd3aa0a48e4650a542269/schema/config-schema.json" --output components/renku_data_services/notebooks/oci/image_config.py --class-name ImageConfig --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS}
poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-index-schema.json" --output components/renku_data_services/notebooks/oci/image_index.py --class-name ImageIndex --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS}
poetry run datamodel-codegen --url "https://raw.githubusercontent.com/opencontainers/image-spec/refs/tags/v1.1.1/schema/image-manifest-schema.json" --output components/renku_data_services/notebooks/oci/image_manifest.py --class-name ImageManifest --base-class renku_data_services.notebooks.oci.base_model.BaseOciModel ${CR_CODEGEN_PARAMS}

##@ Devcontainer

.PHONY: devcontainer_up
Expand Down
16 changes: 8 additions & 8 deletions bases/renku_data_services/data_api/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,19 +205,19 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
url_prefix=url_prefix,
authenticator=dm.authenticator,
nb_config=dm.config.nb_config,
cluster_repo=dm.cluster_repo,
data_connector_repo=dm.data_connector_repo,
data_connector_secret_repo=dm.data_connector_secret_repo,
git_provider_helper=dm.git_provider_helper,
image_check_repo=dm.image_check_repo,
internal_gitlab_authenticator=dm.gitlab_authenticator,
metrics=dm.metrics,
project_repo=dm.project_repo,
project_session_secret_repo=dm.project_session_secret_repo,
rp_repo=dm.rp_repo,
session_repo=dm.session_repo,
storage_repo=dm.storage_repo,
rp_repo=dm.rp_repo,
user_repo=dm.kc_user_repo,
data_connector_repo=dm.data_connector_repo,
data_connector_secret_repo=dm.data_connector_secret_repo,
cluster_repo=dm.cluster_repo,
internal_gitlab_authenticator=dm.gitlab_authenticator,
metrics=dm.metrics,
connected_svcs_repo=dm.connected_services_repo,
git_provider_helper=dm.git_provider_helper,
)
platform_config = PlatformConfigBP(
name="platform_config",
Expand Down
7 changes: 7 additions & 0 deletions bases/renku_data_services/data_api/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
from renku_data_services.notebooks.api.classes.data_service import DummyGitProviderHelper, GitProviderHelper
from renku_data_services.notebooks.config import GitProviderHelperProto, get_clusters
from renku_data_services.notebooks.constants import AMALTHEA_SESSION_GVK, JUPYTER_SESSION_GVK
from renku_data_services.notebooks.image_check import ImageCheckRepository
from renku_data_services.platform.db import PlatformRepository, UrlRedirectRepository
from renku_data_services.project.db import (
ProjectMemberRepository,
Expand Down Expand Up @@ -138,6 +139,7 @@ class DependencyManager:
data_connector_repo: DataConnectorRepository
data_connector_secret_repo: DataConnectorSecretRepository
cluster_repo: ClusterRepository
image_check_repo: ImageCheckRepository
metrics_repo: MetricsRepository
metrics: StagingMetricsService
shipwright_client: ShipwrightClient | None
Expand Down Expand Up @@ -372,6 +374,10 @@ def from_env(cls) -> DependencyManager:
secret_service_public_key=config.secrets.public_key,
authz=authz,
)
image_check_repo = ImageCheckRepository(
nb_config=config.nb_config,
connected_services_repo=connected_services_repo,
)
search_reprovisioning = SearchReprovision(
search_updates_repo=search_updates_repo,
reprovisioning_repo=reprovisioning_repo,
Expand Down Expand Up @@ -410,6 +416,7 @@ def from_env(cls) -> DependencyManager:
data_connector_repo=data_connector_repo,
data_connector_secret_repo=data_connector_secret_repo,
cluster_repo=cluster_repo,
image_check_repo=image_check_repo,
metrics_repo=metrics_repo,
metrics=metrics,
shipwright_client=shipwright_client,
Expand Down
26 changes: 26 additions & 0 deletions components/renku_data_services/notebooks/api.spec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1076,6 +1076,8 @@ components:
accessible:
type: boolean
description: Whether the image is accessible or not.
platforms:
$ref: "#/components/schemas/ImagePlatforms"
connection:
"$ref": "#/components/schemas/ImageConnection"
provider:
Expand Down Expand Up @@ -1114,6 +1116,30 @@ components:
- id
- name
- url
ImagePlatforms:
type: array
description: The list of platforms an image can run on.
items:
$ref: "#/components/schemas/ImagePlatform"
ImagePlatform:
type: object
description: A runtime platform.
properties:
architecture:
type: string
os:
type: string
os.version:
type: string
os.features:
type: array
items:
type: string
variant:
type: string
required:
- architecture
- os
responses:
ImageCheckResponse:
description: Information about whether a docker image is available or not and if there is a connected service then which connected service can be used to access the image.
Expand Down
40 changes: 27 additions & 13 deletions components/renku_data_services/notebooks/api/classes/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,22 +142,43 @@ def platform_matches(manifest: dict[str, Any]) -> bool:

async def image_exists(self, image: Image) -> bool:
"""Check the docker repo API if the image exists."""
return await self.image_check(image) == 200
status_code, _ = await self.image_check(image)
return status_code == 200

async def image_check(self, image: Image) -> int:
async def image_check(self, image: Image, include_manifest: bool = False) -> tuple[int, httpx.Response]:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be a silly question but why not return only the httpx.Response ? The code must be extracted from it anyway.

"""Check the image at the registry."""
token = await self._get_docker_token(image)
image_digest_url = f"{self.scheme}://{image.hostname}/v2/{image.name}/manifests/{image.tag}"
accept_media = ",".join(
[e.value for e in [ManifestTypes.docker_v2, ManifestTypes.oci_v1_manifest, ManifestTypes.oci_v1_index]]
[
e.value
for e in [
ManifestTypes.docker_v2,
ManifestTypes.docker_v2_list,
ManifestTypes.oci_v1_manifest,
ManifestTypes.oci_v1_index,
]
]
)
headers = {"Accept": accept_media}
if token:
headers["Authorization"] = f"Bearer {token}"
method = "GET" if include_manifest else "HEAD"

res = await self.client.head(image_digest_url, headers=headers)
res = await self.client.request(method, image_digest_url, headers=headers)
logger.debug(f"Checked image access: {image_digest_url}: {res.status_code}")
return res.status_code
return res.status_code, res

async def get_image_config_from_digest(self, image: Image, config_digest: str) -> httpx.Response:
"""Query the docker API to get the configuration of an image from the config digest."""
token = await self._get_docker_token(image)
return await self.client.get(
f"{self.scheme}://{image.hostname}/v2/{image.name}/blobs/{config_digest}",
headers={
"Accept": "application/json",
"Authorization": f"Bearer {token}",
},
)

async def get_image_config(self, image: Image) -> Optional[dict[str, Any]]:
"""Query the docker API to get the configuration of an image."""
Expand All @@ -167,14 +188,7 @@ async def get_image_config(self, image: Image) -> Optional[dict[str, Any]]:
config_digest = manifest.get("config", {}).get("digest")
if config_digest is None:
return None
token = await self._get_docker_token(image)
res = await self.client.get(
f"{self.scheme}://{image.hostname}/v2/{image.name}/blobs/{config_digest}",
headers={
"Accept": "application/json",
"Authorization": f"Bearer {token}",
},
)
res = await self.get_image_config_from_digest(image, config_digest)
if res.status_code != 200:
return None
return cast(dict[str, Any], res.json())
Expand Down
13 changes: 12 additions & 1 deletion components/renku_data_services/notebooks/apispec.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: api.spec.yaml
# timestamp: 2025-10-15T12:41:50+00:00
# timestamp: 2025-11-06T12:51:12+00:00

from __future__ import annotations

Expand Down Expand Up @@ -279,6 +279,14 @@ class ImageProvider(BaseAPISpec):
url: str


class ImagePlatform(BaseAPISpec):
architecture: str
os: str
os_version: Optional[str] = Field(None, alias="os.version")
os_features: Optional[List[str]] = Field(None, alias="os.features")
variant: Optional[str] = None


class NotebooksImagesGetParametersQuery(BaseAPISpec):
image_url: str = Field(..., min_length=1)

Expand Down Expand Up @@ -448,6 +456,9 @@ class SessionListResponse(RootModel[List[SessionResponse]]):

class ImageCheckResponse(BaseAPISpec):
accessible: bool = Field(..., description="Whether the image is accessible or not.")
platforms: Optional[List[ImagePlatform]] = Field(
None, description="The list of platforms an image can run on."
)
connection: Optional[ImageConnection] = None
provider: Optional[ImageProvider] = None

Expand Down
37 changes: 21 additions & 16 deletions components/renku_data_services/notebooks/blueprints.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,14 @@
from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
from renku_data_services.base_models import AnonymousAPIUser, APIUser, AuthenticatedAPIUser, Authenticator
from renku_data_services.base_models.metrics import MetricsService
from renku_data_services.connected_services.db import ConnectedServicesRepository
from renku_data_services.connected_services.models import ConnectionStatus
from renku_data_services.crc.db import ClusterRepository, ResourcePoolRepository
from renku_data_services.data_connectors.db import (
DataConnectorRepository,
DataConnectorSecretRepository,
)
from renku_data_services.errors import errors
from renku_data_services.notebooks import apispec, core, image_check
from renku_data_services.notebooks import apispec, core
from renku_data_services.notebooks.api.classes.image import Image
from renku_data_services.notebooks.api.schemas.config_server_options import ServerOptionsEndpointResponse
from renku_data_services.notebooks.api.schemas.logs import ServerLogs
Expand All @@ -31,6 +30,7 @@
validate_session_post_request,
)
from renku_data_services.notebooks.errors.intermittent import AnonymousUserPatchError
from renku_data_services.notebooks.image_check import ImageCheckRepository
from renku_data_services.project.db import ProjectRepository, ProjectSessionSecretRepository
from renku_data_services.session.db import SessionRepository
from renku_data_services.storage.db import StorageRepository
Expand Down Expand Up @@ -196,18 +196,18 @@ class NotebooksNewBP(CustomBlueprint):
authenticator: base_models.Authenticator
internal_gitlab_authenticator: base_models.Authenticator
nb_config: NotebooksConfig
cluster_repo: ClusterRepository
data_connector_repo: DataConnectorRepository
data_connector_secret_repo: DataConnectorSecretRepository
git_provider_helper: GitProviderHelperProto
image_check_repo: ImageCheckRepository
project_repo: ProjectRepository
project_session_secret_repo: ProjectSessionSecretRepository
session_repo: SessionRepository
rp_repo: ResourcePoolRepository
session_repo: SessionRepository
storage_repo: StorageRepository
user_repo: UserRepo
data_connector_repo: DataConnectorRepository
data_connector_secret_repo: DataConnectorSecretRepository
metrics: MetricsService
cluster_repo: ClusterRepository
connected_svcs_repo: ConnectedServicesRepository
git_provider_helper: GitProviderHelperProto

def start(self) -> BlueprintFactoryResponse:
"""Start a session with the new operator."""
Expand Down Expand Up @@ -236,7 +236,7 @@ async def _handler(
session_repo=self.session_repo,
user_repo=self.user_repo,
metrics=self.metrics,
connected_svcs_repo=self.connected_svcs_repo,
image_check_repo=self.image_check_repo,
)
status = 201 if created else 200
return json(session.as_apispec().model_dump(exclude_none=True, mode="json"), status)
Expand Down Expand Up @@ -303,7 +303,7 @@ async def _handler(
rp_repo=self.rp_repo,
session_repo=self.session_repo,
metrics=self.metrics,
connected_svcs_repo=self.connected_svcs_repo,
image_check_repo=self.image_check_repo,
)
return json(new_session.as_apispec().model_dump(exclude_none=True, mode="json"))

Expand Down Expand Up @@ -337,11 +337,10 @@ async def _check_docker_image(
query: apispec.SessionsImagesGetParametersQuery,
) -> JSONResponse:
image = Image.from_path(query.image_url)
result = await image_check.check_image(
image,
user,
self.connected_svcs_repo,
image_check.InternalGitLabConfig(internal_gitlab_user, self.nb_config),
result = await self.image_check_repo.check_image(
user=user,
gitlab_user=internal_gitlab_user,
image=image,
)
logger.info(f"Checked image {query.image_url}: {result}")
conn = None
Expand All @@ -366,7 +365,13 @@ async def _check_docker_image(
id=result.client.id, name=result.client.display_name, url=result.client.url
)

resp = apispec.ImageCheckResponse(accessible=result.accessible, connection=conn, provider=provider)
platforms = None
if result.platforms:
platforms = [apispec.ImagePlatform.model_validate(p) for p in result.platforms]

resp = apispec.ImageCheckResponse(
accessible=result.accessible, platforms=platforms, connection=conn, provider=provider
)

return json(resp.model_dump(exclude_none=True, mode="json"))

Expand Down
19 changes: 9 additions & 10 deletions components/renku_data_services/notebooks/core_sessions.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
from ulid import ULID
from yaml import safe_dump

import renku_data_services.notebooks.image_check as ic
from renku_data_services.app_config import logging
from renku_data_services.base_models import AnonymousAPIUser, APIUser, AuthenticatedAPIUser
from renku_data_services.base_models.metrics import MetricsService
from renku_data_services.connected_services.db import ConnectedServicesRepository
from renku_data_services.crc.db import ClusterRepository, ResourcePoolRepository
from renku_data_services.crc.models import (
ClusterSettings,
Expand Down Expand Up @@ -79,6 +77,7 @@
State,
Storage,
)
from renku_data_services.notebooks.image_check import ImageCheckRepository
from renku_data_services.notebooks.models import (
ExtraSecret,
SessionDataConnectorOverride,
Expand Down Expand Up @@ -563,11 +562,11 @@ def __format_image_pull_secret(secret_name: str, access_token: str, registry_dom


async def __get_connected_services_image_pull_secret(
secret_name: str, connected_svcs_repo: ConnectedServicesRepository, image: str, user: APIUser
secret_name: str, image_check_repo: ImageCheckRepository, image: str, user: APIUser
) -> ExtraSecret | None:
"""Return a secret for accessing the image if one is available for the given user."""
image_parsed = Image.from_path(image)
image_check_result = await ic.check_image(image_parsed, user, connected_svcs_repo, None)
image_check_result = await image_check_repo.check_image(user=user, gitlab_user=None, image=image_parsed)
logger.debug(f"Set pull secret for {image} to connection {image_check_result.image_provider}")
if not image_check_result.token:
return None
Expand All @@ -588,12 +587,12 @@ async def get_image_pull_secret(
nb_config: NotebooksConfig,
user: APIUser,
internal_gitlab_user: APIUser,
connected_svcs_repo: ConnectedServicesRepository,
image_check_repo: ImageCheckRepository,
) -> ExtraSecret | None:
"""Get an image pull secret."""

v2_secret = await __get_connected_services_image_pull_secret(
f"{server_name}-image-secret", connected_svcs_repo, image, user
f"{server_name}-image-secret", image_check_repo, image, user
)
if v2_secret:
return v2_secret
Expand Down Expand Up @@ -672,7 +671,7 @@ async def start_session(
session_repo: SessionRepository,
user_repo: UserRepo,
metrics: MetricsService,
connected_svcs_repo: ConnectedServicesRepository,
image_check_repo: ImageCheckRepository,
) -> tuple[AmaltheaSessionV1Alpha1, bool]:
"""Start an Amalthea session.

Expand Down Expand Up @@ -840,7 +839,7 @@ async def start_session(
nb_config=nb_config,
user=user,
internal_gitlab_user=internal_gitlab_user,
connected_svcs_repo=connected_svcs_repo,
image_check_repo=image_check_repo,
)
if image_secret:
session_extras = session_extras.concat(SessionExtraResources(secrets=[image_secret]))
Expand Down Expand Up @@ -983,7 +982,7 @@ async def patch_session(
project_session_secret_repo: ProjectSessionSecretRepository,
rp_repo: ResourcePoolRepository,
session_repo: SessionRepository,
connected_svcs_repo: ConnectedServicesRepository,
image_check_repo: ImageCheckRepository,
metrics: MetricsService,
) -> AmaltheaSessionV1Alpha1:
"""Patch an Amalthea session."""
Expand Down Expand Up @@ -1119,7 +1118,7 @@ async def patch_session(
image=image,
server_name=server_name,
nb_config=nb_config,
connected_svcs_repo=connected_svcs_repo,
image_check_repo=image_check_repo,
user=user,
internal_gitlab_user=internal_gitlab_user,
)
Expand Down
Loading