From b061bdcceb4895bda865d7ac2e2d9cd453d73f20 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 30 Apr 2025 10:12:10 +0300 Subject: [PATCH 1/5] support for fileset management and example --- datacrunch/containers/containers.py | 52 +++++++++++++++++++ .../examples/containers/fileset_secrets.rst | 9 ++++ docs/source/examples/containers/index.rst | 1 + examples/containers/fileset_secret_example.py | 29 +++++++++++ 4 files changed, 91 insertions(+) create mode 100644 docs/source/examples/containers/fileset_secrets.rst create mode 100644 examples/containers/fileset_secret_example.py diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index 0ec28ae..de16a91 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -4,6 +4,8 @@ creation, updates, deletion, and monitoring of containerized applications. """ +import base64 +import os from dataclasses import dataclass, field from dataclasses_json import dataclass_json, Undefined # type: ignore from typing import List, Optional, Dict, Any @@ -18,6 +20,7 @@ SERVERLESS_COMPUTE_RESOURCES_ENDPOINT = '/serverless-compute-resources' CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT = '/container-registry-credentials' SECRETS_ENDPOINT = '/secrets' +FILESET_SECRETS_ENDPOINT = '/file-secrets' class EnvVarType(str, Enum): @@ -27,6 +30,13 @@ class EnvVarType(str, Enum): SECRET = "secret" +class SecretType(str, Enum): + """Types of secrets that can be set in containers.""" + + GENERIC = "generic" # Regular secret, can be used in env vars + FILESET = "file-secret" # A file secret that can be mounted into the container + + class VolumeMountType(str, Enum): """Types of volume mounts that can be configured for containers.""" @@ -446,10 +456,12 @@ class Secret: Attributes: name: Name of the secret. created_at: Timestamp when the secret was created. + secret_type: Type of the secret. """ name: str created_at: str + secret_type: SecretType @dataclass_json @@ -909,6 +921,7 @@ def get_secrets(self) -> List[Secret]: List[Secret]: List of all secrets. """ response = self.client.get(SECRETS_ENDPOINT) + print(response.json()) return [Secret.from_dict(secret) for secret in response.json()] def create_secret(self, name: str, value: str) -> None: @@ -956,3 +969,42 @@ def delete_registry_credentials(self, credentials_name: str) -> None: """ self.client.delete( f"{CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT}/{credentials_name}") + + def get_fileset_secrets(self) -> List[Secret]: + """Retrieves all fileset secrets. + + Returns: + List of all fileset secrets. + """ + response = self.client.get(FILESET_SECRETS_ENDPOINT) + return [Secret.from_dict(secret) for secret in response.json()] + + def delete_fileset_secret(self, secret_name: str) -> None: + """Deletes a fileset secret. + + Args: + secret_name: Name of the secret to delete. + """ + self.client.delete(f"{FILESET_SECRETS_ENDPOINT}/{secret_name}") + + def create_fileset_secret_from_file_paths(self, secret_name: str, file_paths: List[str]) -> None: + """Creates a new fileset secret. + A fileset secret is a secret that contains several files, + and can be used to mount a directory with the files in a container. + + Args: + secret_name: Name of the secret. + file_paths: List of file paths to include in the secret. + """ + processed_files = [] + for file_path in file_paths: + with open(file_path, "rb") as f: + base64_content = base64.b64encode(f.read()).decode("utf-8") + processed_files.append({ + "file_name": os.path.basename(file_path), + "base64_content": base64_content + }) + self.client.post(FILESET_SECRETS_ENDPOINT, { + "name": secret_name, + "files": processed_files + }) diff --git a/docs/source/examples/containers/fileset_secrets.rst b/docs/source/examples/containers/fileset_secrets.rst new file mode 100644 index 0000000..dc2baee --- /dev/null +++ b/docs/source/examples/containers/fileset_secrets.rst @@ -0,0 +1,9 @@ +Fileset Secrets +=============== + +This example shows how to manage fileset secrets for container deployments in DataCrunch. +Fileset secrets are a way to mount a directory with files into a container. + +.. literalinclude:: ../../../../examples/containers/fileset_secret_example.py + :language: python + :caption: Fileset Secrets \ No newline at end of file diff --git a/docs/source/examples/containers/index.rst b/docs/source/examples/containers/index.rst index b87bb9c..7492b2d 100644 --- a/docs/source/examples/containers/index.rst +++ b/docs/source/examples/containers/index.rst @@ -12,6 +12,7 @@ This section contains examples demonstrating how to work with containers in Data environment_variables registry_credentials secrets + fileset_secrets sglang scaling inference_async diff --git a/examples/containers/fileset_secret_example.py b/examples/containers/fileset_secret_example.py new file mode 100644 index 0000000..dd7b6d3 --- /dev/null +++ b/examples/containers/fileset_secret_example.py @@ -0,0 +1,29 @@ +import os +from datacrunch import DataCrunchClient + +# Fileset secrets are a way to mount sensitive files like API keys, certs, and credentials securely inside a container, without hardcoding them in the image or env vars. + + +# Get client secret and id from environment variables +DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID') +DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET') + +# Initialize the client with your credentials +datacrunch = DataCrunchClient(DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET) + +# Define the secret name and the file paths +SECRET_NAME = "my_fileset_secret" +RELATIVE_FILE_PATH = "./relative-path/file1.txt" +ABSOLUTE_FILE_PATH = "/home/username/absolute-path/file2.json" + +# Create the fileset secret that has 2 files +fileset_secret = datacrunch.containers.create_fileset_secret_from_file_paths( + secret_name=SECRET_NAME, file_paths=[RELATIVE_FILE_PATH, ABSOLUTE_FILE_PATH]) + +# Get the secret +secret = datacrunch.containers.get_fileset_secret( + secret_name=SECRET_NAME) +print(secret) + +# Delete the secret +datacrunch.containers.delete_fileset_secret(secret_name=SECRET_NAME) From de60990360977e24bc4daa3fd6d0e08b99a55a19 Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 30 Apr 2025 10:15:53 +0300 Subject: [PATCH 2/5] changelog entry --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f88cc0..d4228a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,8 @@ Changelog ========= +* Added support for fileset secrets + v1.11.0 (2025-04-28) -------------------- From c6c730582401458a0fcc31f79375b2eed906326b Mon Sep 17 00:00:00 2001 From: Tamir Date: Wed, 30 Apr 2025 10:26:46 +0300 Subject: [PATCH 3/5] fix test --- tests/unit_tests/containers/test_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/containers/test_containers.py b/tests/unit_tests/containers/test_containers.py index 806b670..f434539 100644 --- a/tests/unit_tests/containers/test_containers.py +++ b/tests/unit_tests/containers/test_containers.py @@ -134,7 +134,8 @@ SECRETS_DATA = [ { "name": SECRET_NAME, - "created_at": "2023-01-01T00:00:00+00:00" + "created_at": "2023-01-01T00:00:00+00:00", + "secret_type": "generic" } ] From 41e3cd008d6cec1d4c2a0db0fe84772674829269 Mon Sep 17 00:00:00 2001 From: Tamir Date: Mon, 12 May 2025 11:44:45 +0300 Subject: [PATCH 4/5] small fix to example --- examples/containers/fileset_secret_example.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/containers/fileset_secret_example.py b/examples/containers/fileset_secret_example.py index dd7b6d3..c04ddc0 100644 --- a/examples/containers/fileset_secret_example.py +++ b/examples/containers/fileset_secret_example.py @@ -2,7 +2,7 @@ from datacrunch import DataCrunchClient # Fileset secrets are a way to mount sensitive files like API keys, certs, and credentials securely inside a container, without hardcoding them in the image or env vars. - +# This example demonstrates how to create a fileset secret containing two files from your local filesystem # Get client secret and id from environment variables DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID') @@ -11,8 +11,8 @@ # Initialize the client with your credentials datacrunch = DataCrunchClient(DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET) -# Define the secret name and the file paths -SECRET_NAME = "my_fileset_secret" +# Define the secret name and the file paths from your local filesystem where this script is running +SECRET_NAME = "my-fileset-secret" RELATIVE_FILE_PATH = "./relative-path/file1.txt" ABSOLUTE_FILE_PATH = "/home/username/absolute-path/file2.json" @@ -21,9 +21,8 @@ secret_name=SECRET_NAME, file_paths=[RELATIVE_FILE_PATH, ABSOLUTE_FILE_PATH]) # Get the secret -secret = datacrunch.containers.get_fileset_secret( - secret_name=SECRET_NAME) -print(secret) +secrets = datacrunch.containers.get_fileset_secrets() +print(secrets) # Delete the secret datacrunch.containers.delete_fileset_secret(secret_name=SECRET_NAME) From 728180d0b9ec5b734fda692aa3cd0fdb703ab2aa Mon Sep 17 00:00:00 2001 From: Tamir Date: Mon, 12 May 2025 11:53:39 +0300 Subject: [PATCH 5/5] added example for adding fileset volume on deployment creation --- examples/containers/container_deployments_example.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/containers/container_deployments_example.py b/examples/containers/container_deployments_example.py index 38a4877..f91f330 100644 --- a/examples/containers/container_deployments_example.py +++ b/examples/containers/container_deployments_example.py @@ -96,9 +96,16 @@ def main() -> None: path="/health" ), volume_mounts=[ + # Shared memory volume VolumeMount( type=VolumeMountType.SCRATCH, mount_path="/data" + ), + # Fileset secret + VolumeMount( + type=VolumeMountType.SECRET, + mount_path="/path/to/mount", + name="my-fileset-secret" # This fileset secret must be created beforehand ) ], env=[