diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7880b0a..cc3958e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -61,11 +61,6 @@ jobs:
out="${out} catalog-explorer"
fi
- # experiment-tracker
- if git diff --name-only $BASE $HEAD | grep -q '^src/experiment-tracker/'; then
- out="${out} experiment-tracker"
- fi
-
# workspace
if git diff --name-only $BASE $HEAD | grep -q '^src/workspace/'; then
out="${out} workspace"
@@ -109,7 +104,7 @@ jobs:
REG=flintml
OLD=${{ steps.check_version.outputs.old }}
NEW=${{ steps.check_version.outputs.new }}
- all=( storage compute-manager experiment-server catalog-explorer experiment-tracker workspace reverse-proxy worker-base )
+ all=( storage compute-manager experiment-server catalog-explorer workspace reverse-proxy worker-base )
read -r -a changed <<< "${{ steps.detect.outputs.services }}"
for svc in "${all[@]}"; do
@@ -143,10 +138,6 @@ jobs:
CTX="src"
DOCKERFILE="src/catalog-explorer/Dockerfile"
;;
- experiment-tracker)
- CTX="src/experiment-tracker"
- DOCKERFILE="src/experiment-tracker/Dockerfile"
- ;;
workspace)
CTX="src"
DOCKERFILE="src/workspace/Dockerfile"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ad3e720..9347df8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,22 @@
# Changelog
-## [0.1.25] - ...
+## [0.2.0]
+- The objective of this release is to support mounting the Flint Metastore as a POSIX-like filesystem. Consequently, reducing the number of Docker volumes required by the Control Plane.
+- The Experiment Tracker has been collapsed into the Experiment Server for simplicity and improved robustness around `inotify` events.
+- The Experiment Server now stores metrics in the Flint Metastore via JuiceFS, which maintains a metadata database as a SQLite file inside the `storage_meta` mount.
+- The Workspace now mounts the Flint Metastore to store workspace files such as notebooks.
+- Worker Containers also now mount the Flint Metastore, giving them visibility of workspace files. This enables notebooks to be executed within a notebook.
+
+## [0.1.25]
- Redesigned networking to ensure control plane services communicate directly and not via `reverse-proxy`. This avoids circular service dependencies where `reverse-proxy` defines a route for `serviceX` and thus depends on `serviceX`, but `serviceX` depends on `serviceY` and tries to communicate with `serviceY` via `reverse-proxy` that is dependent on `serviceX`...
-## [0.1.24] - ...
+## [0.1.24]
- Added named volume for `workspace`.
-## [0.1.23] - ...
+## [0.1.23]
- First distributed release of FlintML.
...
-## [0.1.0] - ...
+## [0.1.0]
- First build release of FlintML.
diff --git a/README.md b/README.md
index 2546a66..952ab32 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
- 
+ 
@@ -53,7 +53,7 @@ To get a sense of what you can do with FlintML, check out the [Instacart Kaggle
### Data Storage
-The `docker-compose.*.yml` in each FlintML release contains the named Docker volumes `storage_data`, `storage_meta`, `experiment_data` and `workspace_data`. If you wish to specify custom volumes, you should create an override `docker-compose.override.yml` and compose it when spinning up flint. See the [docs](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/).
+FlintML ships with its own [Storage](docs/concepts.md#flint-control-plane) service that depends on the mounts, `storage_data` and `storage_meta`. If you wish to specify custom volumes, you should create an override `docker-compose.override.yml` and compose it when spinning up flint. See the [docs](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/).
### Environment Variables
diff --git a/VERSION b/VERSION
index 20124ad..341cf11 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1.25
\ No newline at end of file
+0.2.0
\ No newline at end of file
diff --git a/docs/concepts.md b/docs/concepts.md
index e897c94..3d9836b 100644
--- a/docs/concepts.md
+++ b/docs/concepts.md
@@ -4,14 +4,14 @@
FlintML contains all the necessary components to enable end-to-end machine learning workloads. It accomplishes this by running several Docker Compose services, constituting the control plane. The control plane network is managed by an internal [nginx](https://github.com/nginx/nginx) container. The key services are:
-- **Storage:** Powered by [Zenko CloudServer](https://github.com/scality/cloudserver), the Storage service houses the *data layer* upon which the [Flint Catalog](#flint-catalog) is built.
-- **Workspace:** The Workspace service serves the FlintML user interface (JupyterLab skin + custom extensions.) A custom [KernelProvisioner](https://jupyter-client.readthedocs.io/en/latest/provisioning.html) communicates with the Compute Manager service.
+- **Storage:** Powered by [Zenko CloudServer](https://github.com/scality/cloudserver), the Storage service comprises the *data layer* upon which the [Flint Catalog](#flint-catalog) is built. This data layer is referred to as the Flint Metastore. Data is stored per the `storage_data` and `storage_meta` Docker volumes.
+- **Workspace:** The Workspace service serves the FlintML user interface (JupyterLab skin + custom extensions.) A custom [KernelProvisioner](https://jupyter-client.readthedocs.io/en/latest/provisioning.html) communicates with the Compute Manager service. Mounts the Flint Metastore using [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) and uses this mount as the JupyterLab working directory.
- **Compute Manager:** The Compute Manager orchestrates and controls all [Worker Containers](#worker-containers) via a configurable *driver*. All requests to start and stop Worker Containers are handled by this service.
-- **Experiment Server:** Integrated with [Aim](https://github.com/aimhubio/aim), the Experiment Server acts as the controller for all ML experiments. It does NOT use the Storage service because it requires a filesystem backend. Thus, an `experiment_data` volume is required. Artifacts are referenced in experiment runs but stored as Objects in the Flint Catalog.
+- **Experiment Server:** Integrated with [Aim](https://github.com/aimhubio/aim), the Experiment Server acts as the controller for all ML experiments and serves the Aim UI. Metrics live in the Flint Metastore as chunks using [JuiceFS](https://juicefs.com/en/). JuiceFS maintains its metadata with a SQLite database inside the `storage_meta` volume, coupling the lifetime of metrics metadata to the liftetime of metrics data.
## Flint Catalog
-The Flint Catalog is a novel and simplified approach to a data catalog. It provides a logical repository that sits on top of the physical locations of files in Storage. This enables key capabilities around governance, search, discovery, lineage and reusability. FlintML will continue to deploy new such capabilities throughout development.
+The Flint Catalog is a novel and simplified approach to a data catalog. It provides a logical repository that sits on top of the physical locations of files in the Flint Metastore. This enables key capabilities around governance, search, discovery, lineage and reusability. FlintML will continue to deploy new such capabilities throughout development. **The Flint Catalog does not govern workspace files or metrics.**
The Flint Catalog defines an Item as logically being a Table (i.e. Delta), or an Object (incl. Artifact sub-type.) All Items are unified within the catalog and considered first-class data-citizens. Rather than logically grouping Items by way of a hierarchical structure like `..` as used in the Unity Catalog, the Flint Catalog allows for full flexibility through the use of tags.
@@ -23,7 +23,7 @@ Importantly, Items are identified by their URI; an Item's URI is a url encoding
FlintML's Control Plane does not execute user code (i.e. development notebooks and workflows.) This work gets delegated to Worker Containers. A Worker Container is a Docker container installed with the `flintml/worker-base` image (directly or indirectly by a derivative image.)
-Each Worker Container runs a single Jupyter kernel that communicates with the Workspace, Storage and Experiment Server services. **Therefore, it is crucial that the host running Worker Containers has networking to the host of the Control Plane.**
+Each Worker Container runs a single Jupyter kernel that communicates with the Workspace, Storage and Experiment Server services. **Therefore, it is crucial that the host running Worker Containers has networking to the host of the Control Plane.** Each Worker Container also mounts Workspace files to its working directory. This enables you to, for example, execute notebooks within a notebook: `%run ./root-level-notebook.ipynb`.
Worker Containers are instantiated by the [Driver](#drivers) configured for use by the Compute Manager service.
diff --git a/examples/instacart.ipynb b/examples/instacart.ipynb
index a46fd1f..57bdcd7 100644
--- a/examples/instacart.ipynb
+++ b/examples/instacart.ipynb
@@ -216,7 +216,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"id": "39081859-c5e0-4106-b67f-2c88824d36f7",
"metadata": {},
"outputs": [
@@ -242,10 +242,11 @@
" name=\"order_products__prior\",\n",
" tags={\n",
" \"source\": \"external\",\n",
- " \"provider\": \"kaggle\"\n",
+ " \"provider\": \"kaggle\",\n",
+ " \"example\": \"instacart\"\n",
" }\n",
")\n",
- "products = scan_delta(path=\"products?provider=kaggle&source=external\") # or copy table path from Catalog Explorer\n",
+ "products = scan_delta(path=\"products?provider=kaggle&source=external&example=instacart\") # or copy table path from Catalog Explorer\n",
"\n",
"# Lazy join\n",
"top_products = (\n",
@@ -277,18 +278,18 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"id": "f39330a1-15d8-4380-8fc5-028b0da868c3",
"metadata": {},
"outputs": [],
"source": [
"# Lazy load all tables\n",
- "aisles = scan_delta(\"aisles?provider=kaggle&source=external\")\n",
- "departments = scan_delta(\"departments?provider=kaggle&source=external\")\n",
- "prior = scan_delta(\"order_products__prior?provider=kaggle&source=external\")\n",
- "train = scan_delta(\"order_products__train?provider=kaggle&source=external\")\n",
- "orders = scan_delta(\"orders?provider=kaggle&source=external\")\n",
- "products = scan_delta(\"products?provider=kaggle&source=external\")\n",
+ "aisles = scan_delta(\"aisles?provider=kaggle&source=external&example=instacart\")\n",
+ "departments = scan_delta(\"departments?provider=kaggle&source=external&example=instacart\")\n",
+ "prior = scan_delta(\"order_products__prior?provider=kaggle&source=external&example=instacart\")\n",
+ "train = scan_delta(\"order_products__train?provider=kaggle&source=external&example=instacart\")\n",
+ "orders = scan_delta(\"orders?provider=kaggle&source=external&example=instacart\")\n",
+ "products = scan_delta(\"products?provider=kaggle&source=external&example=instacart\")\n",
"\n",
"# Join prior orders with order metadata\n",
"prior_full = (\n",
@@ -433,7 +434,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"id": "91a63c9d-3317-4cf8-bc34-b665381e9f65",
"metadata": {},
"outputs": [],
@@ -443,7 +444,7 @@
"from sklearn.preprocessing import LabelEncoder\n",
"\n",
"# Load full dataset into memory\n",
- "df = read_delta(\"features?env=dev\")\n",
+ "df = read_delta(\"features?env=dev&example=instacart\")\n",
"df = df.drop([\"product_name\"])\n",
"\n",
"# Encode categorical columns\n",
@@ -471,7 +472,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"id": "0e070ada-620c-4d96-a491-f18502f3ae93",
"metadata": {},
"outputs": [
@@ -1469,9 +1470,9 @@
"run = new_run(experiment=\"xgb-reorder-predictor\")\n",
"run[\"hparams\"] = {\n",
" \"scale_pos_weight\": ratio,\n",
- " \"n_estimators\": 100,\n",
+ " \"n_estimators\": 20,\n",
" \"learning_rate\": 0.1,\n",
- " \"max_depth\": 6,\n",
+ " \"max_depth\": 4,\n",
"}\n",
"\n",
"# Define model\n",
@@ -1480,9 +1481,9 @@
" scale_pos_weight=ratio,\n",
" eval_metric=\"logloss\",\n",
" use_label_encoder=False,\n",
- " n_estimators=100,\n",
+ " n_estimators=20,\n",
" learning_rate=0.1,\n",
- " max_depth=6,\n",
+ " max_depth=4,\n",
" random_state=42\n",
")\n",
"\n",
@@ -1619,14 +1620,13 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"id": "6a525197-1426-494c-9926-5ff070d6e249",
"metadata": {},
"outputs": [],
"source": [
"from flint import drop_delta, delete_object\n",
"\n",
- "delete_object(\"untrained-model?example=instacart&proj=instacart&run_id=cc31f135d4ff4c04a2eb53dd\")\n",
"drop_delta(\"features?env=dev&example=instacart\")\n",
"drop_delta(\"order_products__train?example=instacart&provider=kaggle&source=external\")\n",
"drop_delta(\"departments?example=instacart&provider=kaggle&source=external\")\n",
@@ -1642,7 +1642,9 @@
"id": "c5881504-29e7-45be-9999-127216128423",
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "delete_object(\"untrained-model?example=instacart&proj=instacart&run_id=93a02bd3348846edb3e6179c\") # You will have a different run_id"
+ ]
}
],
"metadata": {
diff --git a/src/compute-manager/Dockerfile-old b/src/compute-manager/Dockerfile-old
deleted file mode 100644
index 9dbfb8b..0000000
--- a/src/compute-manager/Dockerfile-old
+++ /dev/null
@@ -1,28 +0,0 @@
-FROM python:3.12-slim
-
-WORKDIR /app
-
-# Install system packages: curl, ping (iputils-ping), ssh client
-RUN apt-get update && \
- apt-get install -y --no-install-recommends \
- curl \
- iputils-ping \
- openssh-client && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
-
-COPY ./ ./
-
-RUN chmod +x entrypoint.sh
-
-RUN pip install --no-cache-dir poetry
-RUN poetry config virtualenvs.create false
-RUN poetry install --no-root
-
-ENTRYPOINT ["/app/entrypoint.sh"]
-
-HEALTHCHECK --interval=30s \
- --timeout=5s \
- --start-period=5s \
- --retries=3 \
- CMD curl --fail http://127.0.0.1:8000/health || exit 1
\ No newline at end of file
diff --git a/src/compute-manager/src/driver/local.py b/src/compute-manager/src/driver/local.py
index 9277fd8..fbc79ac 100644
--- a/src/compute-manager/src/driver/local.py
+++ b/src/compute-manager/src/driver/local.py
@@ -2,6 +2,7 @@
import docker
from docker.models.containers import Container
from docker.errors import NotFound, APIError
+from docker.types import LogConfig
from typing import Dict
import os
from typing import Tuple, Optional
@@ -73,11 +74,10 @@ async def launch_container(self, ctx: ContainerContext) -> None:
# Launch the container (do NOT start ipykernel yet)
container = await asyncio.to_thread(
- self._docker.containers.run,
+ self._docker.containers.create,
image=self.worker_image,
- name=f"flint__{project_name}__worker__{ctx.id}",
- detach=True,
auto_remove=True,
+ name=f"flint__{project_name}__worker__{ctx.id}",
network=network_name,
labels={"flint.ephemeral": "true"},
environment={
@@ -87,20 +87,19 @@ async def launch_container(self, ctx: ContainerContext) -> None:
"STORAGE_USER": os.environ.get("STORAGE_USER"),
"STORAGE_PASSWORD": os.environ.get("STORAGE_PASSWORD")
},
- command=["sh", "-c", "poetry run python /root/watchdog.py >> /tmp/watchdog.log 2>&1"],
tty=True,
stdin_open=True,
volumes=volumes_dict,
+ devices=["/dev/fuse:/dev/fuse"],
+ cap_add=["SYS_ADMIN"],
+ security_opt=["apparmor:unconfined"],
+ log_config=LogConfig(type="local")
)
# Copy connection.json into container
await asyncio.to_thread(container.put_archive, "/tmp", tarstream.read())
- # Run ipykernel inside the container
- run_kernel_cmd = [
- "poetry", "run", "python", "-m", "ipykernel_launcher", "-f", "/tmp/connection.json"
- ]
- await asyncio.to_thread(container.exec_run, run_kernel_cmd, detach=True)
+ await asyncio.to_thread(container.start)
# Update container context
await asyncio.to_thread(container.reload)
diff --git a/src/docker-compose.build.yml b/src/docker-compose.build.yml
index 5a752d2..5c9ce5b 100644
--- a/src/docker-compose.build.yml
+++ b/src/docker-compose.build.yml
@@ -3,6 +3,14 @@ x-storage-creds: &storage-creds
STORAGE_USER: ${STORAGE_USER:-admin}
STORAGE_PASSWORD: ${STORAGE_PASSWORD:-password}
+x-s3fs: &s3fs
+ devices:
+ - /dev/fuse:/dev/fuse
+ cap_add:
+ - SYS_ADMIN
+ security_opt:
+ - apparmor:unconfined
+
services:
# --- BACKEND SERVICES ---
@@ -24,7 +32,7 @@ services:
restart: always
experiment-server:
- <<: *storage-creds
+ <<: [*storage-creds, *s3fs]
build:
context: .
dockerfile: ./experiment-server/Dockerfile
@@ -32,7 +40,7 @@ services:
storage:
condition: service_healthy
volumes:
- - experiment_data:/repo
+ - storage_meta:/meta
restart: always
### --- FRONTEND SERVICES ---
@@ -46,26 +54,14 @@ services:
storage:
condition: service_healthy
restart: always
-
- experiment-tracker:
- build: ./experiment-tracker
- depends_on:
- experiment-server:
- condition: service_healthy
- volumes:
- - experiment_data:/repo
- restart: always
workspace:
- <<: *storage-creds
+ <<: [*storage-creds, *s3fs]
build:
context: .
dockerfile: ./workspace/Dockerfile
depends_on:
- catalog-explorer
- - experiment-tracker
- volumes:
- - workspace_data:/srv/workspace
restart: always
reverse-proxy:
@@ -81,14 +77,10 @@ services:
condition: service_started
catalog-explorer:
condition: service_started
- experiment-tracker:
- condition: service_started
ports:
- "${FLINT_PORT:-8701}:80"
restart: always
volumes:
storage_data:
- storage_meta:
- experiment_data:
- workspace_data:
\ No newline at end of file
+ storage_meta:
\ No newline at end of file
diff --git a/src/docker-compose.release-template.yml b/src/docker-compose.release-template.yml
index 9ef3c15..8d7c623 100644
--- a/src/docker-compose.release-template.yml
+++ b/src/docker-compose.release-template.yml
@@ -3,6 +3,14 @@ x-storage-creds: &storage-creds
STORAGE_USER: ${STORAGE_USER:-admin}
STORAGE_PASSWORD: ${STORAGE_PASSWORD:-password}
+x-s3fs: &s3fs
+ devices:
+ - /dev/fuse:/dev/fuse
+ cap_add:
+ - SYS_ADMIN
+ security_opt:
+ - apparmor:unconfined
+
services:
# --- BACKEND SERVICES ---
@@ -24,13 +32,13 @@ services:
restart: always
experiment-server:
- <<: *storage-creds
+ <<: [*storage-creds, *s3fs]
image: flintml/experiment-server:${VERSION}
depends_on:
storage:
condition: service_healthy
volumes:
- - experiment_data:/repo
+ - storage_meta:/meta
restart: always
### --- FRONTEND SERVICES ---
@@ -43,23 +51,11 @@ services:
condition: service_healthy
restart: always
- experiment-tracker:
- image: flintml/experiment-tracker:${VERSION}
- depends_on:
- experiment-server:
- condition: service_healthy
- volumes:
- - experiment_data:/repo
- restart: always
-
workspace:
- <<: *storage-creds
+ <<: [*storage-creds, *s3fs]
image: flintml/workspace:${VERSION}
depends_on:
- catalog-explorer
- - experiment-tracker
- volumes:
- - workspace_data:/srv/workspace
restart: always
reverse-proxy:
@@ -75,14 +71,10 @@ services:
condition: service_started
catalog-explorer:
condition: service_started
- experiment-tracker:
- condition: service_started
ports:
- "${FLINT_PORT:-8701}:80"
restart: always
volumes:
storage_data:
- storage_meta:
- experiment_data:
- workspace_data:
\ No newline at end of file
+ storage_meta:
\ No newline at end of file
diff --git a/src/experiment-server/Dockerfile b/src/experiment-server/Dockerfile
index 1d07adb..0f79e37 100644
--- a/src/experiment-server/Dockerfile
+++ b/src/experiment-server/Dockerfile
@@ -1,15 +1,26 @@
FROM aimstack/aim
+ENV FLINT_CONTROL_PLANE_ENDPOINT=http://reverse-proxy
+ENV META_PATH=/meta/juicefs/juice_meta.db
+
WORKDIR /app
COPY ./experiment-server ./experiment-server
+# Configure JuiceFS
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ fuse libfuse2 curl ca-certificates xz-utils tar && \
+ curl -fsSL https://d.juicefs.com/install | sh - && \
+ rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /mnt/metastore
+
RUN mv ./experiment-server/entrypoint.sh /root/entrypoint.sh
RUN chmod +x /root/entrypoint.sh
ENTRYPOINT ["/root/entrypoint.sh"]
-HEALTHCHECK --interval=5s \
+HEALTHCHECK --interval=10s \
--timeout=3s \
--start-period=5s \
- --retries=5 \
+ --retries=6 \
CMD aim runs --repo aim://localhost:53800 ls || exit 1
\ No newline at end of file
diff --git a/src/experiment-server/entrypoint.sh b/src/experiment-server/entrypoint.sh
index abfdee4..c811697 100644
--- a/src/experiment-server/entrypoint.sh
+++ b/src/experiment-server/entrypoint.sh
@@ -1,13 +1,43 @@
#!/bin/bash
set -e
-# Check if /repo is empty
-if [ -z "$(ls -A /repo)" ]; then
- echo "🔨 /repo is empty — initializing Aim repo…"
- aim init --repo /repo
-else
- echo "✔️ /repo is not empty — skipping init."
+# Configure JucieFS
+export AWS_ACCESS_KEY_ID="${STORAGE_USER}"
+export AWS_SECRET_ACCESS_KEY="${STORAGE_PASSWORD}"
+export AWS_REGION=us-east-1
+export AWS_S3_FORCE_PATH_STYLE=1
+
+mkdir -p "$(dirname "$META_PATH")"
+
+META_URL="sqlite3://$META_PATH"
+
+juicefs format \
+ --storage s3 \
+ --bucket http://storage:8000/metastore \
+ "$META_URL" experiment
+
+mkdir -p /mnt/metastore/experiment
+juicefs mount "$META_URL" /mnt/metastore/experiment -d
+
+echo "Checking if Aim repo exists"
+
+# Initialise repo if new
+if [[ ! -d /mnt/metastore/experiment/.aim ]]; then
+ echo "Aim repo does not exist creating a new one"
+ aim init --repo /mnt/metastore/experiment
fi
-echo "🚀 Launching Aim server…"
-exec aim server --repo /repo --host 0.0.0.0 --port 53800
+echo "Starting Aim"
+
+# Start Aim
+aim server \
+ --repo /mnt/metastore/experiment \
+ --host 0.0.0.0 --port 53800 &
+
+server_pid=$!
+
+aim up \
+ --repo /mnt/metastore/experiment \
+ --host 0.0.0.0 --port 43800 \
+ --base-path /experiment-tracker \
+ --log-level debug
\ No newline at end of file
diff --git a/src/experiment-tracker/Dockerfile b/src/experiment-tracker/Dockerfile
deleted file mode 100644
index e0a7c66..0000000
--- a/src/experiment-tracker/Dockerfile
+++ /dev/null
@@ -1,7 +0,0 @@
-FROM aimstack/aim:3.29.1
-
-COPY ./ ./
-
-RUN mv ./entrypoint.sh /root/entrypoint.sh
-RUN chmod +x /root/entrypoint.sh
-ENTRYPOINT ["/root/entrypoint.sh"]
\ No newline at end of file
diff --git a/src/experiment-tracker/entrypoint.sh b/src/experiment-tracker/entrypoint.sh
deleted file mode 100644
index 79055b5..0000000
--- a/src/experiment-tracker/entrypoint.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-set -e
-
-aim up --repo /repo --host 0.0.0.0 --port 43800 --base-path /experiment-tracker
\ No newline at end of file
diff --git a/src/reverse-proxy/default.conf b/src/reverse-proxy/default.conf
index 0a54210..8dbbfd9 100644
--- a/src/reverse-proxy/default.conf
+++ b/src/reverse-proxy/default.conf
@@ -45,7 +45,7 @@ server {
}
location /experiment-tracker/ {
- proxy_pass http://experiment-tracker:43800/experiment-tracker/;
+ proxy_pass http://experiment-server:43800/experiment-tracker/;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
diff --git a/src/storage/config.json b/src/storage/config.json
index 7e4fbdb..44a2b08 100644
--- a/src/storage/config.json
+++ b/src/storage/config.json
@@ -60,8 +60,8 @@
},
"clusters": 1,
"log": {
- "logLevel": "trace",
- "dumpLevel": "trace"
+ "logLevel": "info",
+ "dumpLevel": "info"
},
"healthChecks": {
"allowFrom": ["127.0.0.1/8", "::1"]
diff --git a/src/worker-base/Dockerfile b/src/worker-base/Dockerfile
index dc79701..d65e8a6 100644
--- a/src/worker-base/Dockerfile
+++ b/src/worker-base/Dockerfile
@@ -7,6 +7,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
nano \
procps \
net-tools \
+ s3fs \
+ fuse \
+ libfuse2 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -27,4 +30,11 @@ RUN poetry install --no-root
COPY ./worker-base/startup/watchdog.py /root/watchdog.py
RUN mkdir -p /root/.ipython/profile_default/startup
-COPY ./worker-base/startup/00-flint-init.py /root/.ipython/profile_default/startup/00-flint-init.py
\ No newline at end of file
+COPY ./worker-base/startup/00-flint-init.py /root/.ipython/profile_default/startup/00-flint-init.py
+
+# Create S3FS mount point
+RUN mkdir /mnt/metastore
+
+RUN mv /app/worker-base/startup/entrypoint.sh /root/entrypoint.sh
+RUN chmod +x /root/entrypoint.sh
+ENTRYPOINT ["/root/entrypoint.sh"]
\ No newline at end of file
diff --git a/src/worker-base/poetry.lock b/src/worker-base/poetry.lock
index 380ac21..3115e86 100644
--- a/src/worker-base/poetry.lock
+++ b/src/worker-base/poetry.lock
@@ -940,6 +940,21 @@ typing-extensions = ">=4.8.0"
all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"]
+[[package]]
+name = "fastjsonschema"
+version = "2.21.1"
+description = "Fastest Python implementation of JSON schema"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+ {file = "fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667"},
+ {file = "fastjsonschema-2.21.1.tar.gz", hash = "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4"},
+]
+
+[package.extras]
+devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"]
+
[[package]]
name = "filelock"
version = "3.18.0"
@@ -1420,6 +1435,43 @@ files = [
{file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
]
+[[package]]
+name = "jsonschema"
+version = "4.24.0"
+description = "An implementation of JSON Schema validation for Python"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d"},
+ {file = "jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196"},
+]
+
+[package.dependencies]
+attrs = ">=22.2.0"
+jsonschema-specifications = ">=2023.03.6"
+referencing = ">=0.28.4"
+rpds-py = ">=0.7.1"
+
+[package.extras]
+format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"]
+format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"]
+
+[[package]]
+name = "jsonschema-specifications"
+version = "2025.4.1"
+description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "jsonschema_specifications-2025.4.1-py3-none-any.whl", hash = "sha256:4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af"},
+ {file = "jsonschema_specifications-2025.4.1.tar.gz", hash = "sha256:630159c9f4dbea161a6a2205c3011cc4f18ff381b189fff48bb39b9bf26ae608"},
+]
+
+[package.dependencies]
+referencing = ">=0.31.0"
+
[[package]]
name = "jupyter-client"
version = "8.6.3"
@@ -1696,6 +1748,28 @@ files = [
{file = "multidict-6.4.4.tar.gz", hash = "sha256:69ee9e6ba214b5245031b76233dd95408a0fd57fdb019ddcc1ead4790932a8e8"},
]
+[[package]]
+name = "nbformat"
+version = "5.10.4"
+description = "The Jupyter Notebook format"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"},
+ {file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"},
+]
+
+[package.dependencies]
+fastjsonschema = ">=2.15"
+jsonschema = ">=2.6"
+jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0"
+traitlets = ">=5.1"
+
+[package.extras]
+docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"]
+test = ["pep440", "pre-commit", "pytest", "testpath"]
+
[[package]]
name = "nest-asyncio"
version = "1.6.0"
@@ -2665,6 +2739,23 @@ files = [
[package.dependencies]
cffi = {version = "*", markers = "implementation_name == \"pypy\""}
+[[package]]
+name = "referencing"
+version = "0.36.2"
+description = "JSON Referencing + Python"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0"},
+ {file = "referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa"},
+]
+
+[package.dependencies]
+attrs = ">=22.2.0"
+rpds-py = ">=0.7.0"
+typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""}
+
[[package]]
name = "requests"
version = "2.32.3"
@@ -2703,6 +2794,133 @@ files = [
docs = ["Sphinx", "furo"]
test = ["pytest", "pytest-mock"]
+[[package]]
+name = "rpds-py"
+version = "0.25.1"
+description = "Python bindings to Rust's persistent data structures (rpds)"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "rpds_py-0.25.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f4ad628b5174d5315761b67f212774a32f5bad5e61396d38108bd801c0a8f5d9"},
+ {file = "rpds_py-0.25.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c742af695f7525e559c16f1562cf2323db0e3f0fbdcabdf6865b095256b2d40"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:605ffe7769e24b1800b4d024d24034405d9404f0bc2f55b6db3362cd34145a6f"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ccc6f3ddef93243538be76f8e47045b4aad7a66a212cd3a0f23e34469473d36b"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f70316f760174ca04492b5ab01be631a8ae30cadab1d1081035136ba12738cfa"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1dafef8df605fdb46edcc0bf1573dea0d6d7b01ba87f85cd04dc855b2b4479e"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0701942049095741a8aeb298a31b203e735d1c61f4423511d2b1a41dcd8a16da"},
+ {file = "rpds_py-0.25.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e87798852ae0b37c88babb7f7bbbb3e3fecc562a1c340195b44c7e24d403e380"},
+ {file = "rpds_py-0.25.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3bcce0edc1488906c2d4c75c94c70a0417e83920dd4c88fec1078c94843a6ce9"},
+ {file = "rpds_py-0.25.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e2f6a2347d3440ae789505693a02836383426249d5293541cd712e07e7aecf54"},
+ {file = "rpds_py-0.25.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4fd52d3455a0aa997734f3835cbc4c9f32571345143960e7d7ebfe7b5fbfa3b2"},
+ {file = "rpds_py-0.25.1-cp310-cp310-win32.whl", hash = "sha256:3f0b1798cae2bbbc9b9db44ee068c556d4737911ad53a4e5093d09d04b3bbc24"},
+ {file = "rpds_py-0.25.1-cp310-cp310-win_amd64.whl", hash = "sha256:3ebd879ab996537fc510a2be58c59915b5dd63bccb06d1ef514fee787e05984a"},
+ {file = "rpds_py-0.25.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5f048bbf18b1f9120685c6d6bb70cc1a52c8cc11bdd04e643d28d3be0baf666d"},
+ {file = "rpds_py-0.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fbb0dbba559959fcb5d0735a0f87cdbca9e95dac87982e9b95c0f8f7ad10255"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4ca54b9cf9d80b4016a67a0193ebe0bcf29f6b0a96f09db942087e294d3d4c2"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ee3e26eb83d39b886d2cb6e06ea701bba82ef30a0de044d34626ede51ec98b0"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89706d0683c73a26f76a5315d893c051324d771196ae8b13e6ffa1ffaf5e574f"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2013ee878c76269c7b557a9a9c042335d732e89d482606990b70a839635feb7"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45e484db65e5380804afbec784522de84fa95e6bb92ef1bd3325d33d13efaebd"},
+ {file = "rpds_py-0.25.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:48d64155d02127c249695abb87d39f0faf410733428d499867606be138161d65"},
+ {file = "rpds_py-0.25.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:048893e902132fd6548a2e661fb38bf4896a89eea95ac5816cf443524a85556f"},
+ {file = "rpds_py-0.25.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0317177b1e8691ab5879f4f33f4b6dc55ad3b344399e23df2e499de7b10a548d"},
+ {file = "rpds_py-0.25.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bffcf57826d77a4151962bf1701374e0fc87f536e56ec46f1abdd6a903354042"},
+ {file = "rpds_py-0.25.1-cp311-cp311-win32.whl", hash = "sha256:cda776f1967cb304816173b30994faaf2fd5bcb37e73118a47964a02c348e1bc"},
+ {file = "rpds_py-0.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:dc3c1ff0abc91444cd20ec643d0f805df9a3661fcacf9c95000329f3ddf268a4"},
+ {file = "rpds_py-0.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:5a3ddb74b0985c4387719fc536faced33cadf2172769540c62e2a94b7b9be1c4"},
+ {file = "rpds_py-0.25.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b5ffe453cde61f73fea9430223c81d29e2fbf412a6073951102146c84e19e34c"},
+ {file = "rpds_py-0.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:115874ae5e2fdcfc16b2aedc95b5eef4aebe91b28e7e21951eda8a5dc0d3461b"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a714bf6e5e81b0e570d01f56e0c89c6375101b8463999ead3a93a5d2a4af91fa"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:35634369325906bcd01577da4c19e3b9541a15e99f31e91a02d010816b49bfda"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4cb2b3ddc16710548801c6fcc0cfcdeeff9dafbc983f77265877793f2660309"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9ceca1cf097ed77e1a51f1dbc8d174d10cb5931c188a4505ff9f3e119dfe519b"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2cd1a4b0c2b8c5e31ffff50d09f39906fe351389ba143c195566056c13a7ea"},
+ {file = "rpds_py-0.25.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1de336a4b164c9188cb23f3703adb74a7623ab32d20090d0e9bf499a2203ad65"},
+ {file = "rpds_py-0.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9fca84a15333e925dd59ce01da0ffe2ffe0d6e5d29a9eeba2148916d1824948c"},
+ {file = "rpds_py-0.25.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:88ec04afe0c59fa64e2f6ea0dd9657e04fc83e38de90f6de201954b4d4eb59bd"},
+ {file = "rpds_py-0.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a8bd2f19e312ce3e1d2c635618e8a8d8132892bb746a7cf74780a489f0f6cdcb"},
+ {file = "rpds_py-0.25.1-cp312-cp312-win32.whl", hash = "sha256:e5e2f7280d8d0d3ef06f3ec1b4fd598d386cc6f0721e54f09109a8132182fbfe"},
+ {file = "rpds_py-0.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:db58483f71c5db67d643857404da360dce3573031586034b7d59f245144cc192"},
+ {file = "rpds_py-0.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:6d50841c425d16faf3206ddbba44c21aa3310a0cebc3c1cdfc3e3f4f9f6f5728"},
+ {file = "rpds_py-0.25.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:659d87430a8c8c704d52d094f5ba6fa72ef13b4d385b7e542a08fc240cb4a559"},
+ {file = "rpds_py-0.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:68f6f060f0bbdfb0245267da014d3a6da9be127fe3e8cc4a68c6f833f8a23bb1"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083a9513a33e0b92cf6e7a6366036c6bb43ea595332c1ab5c8ae329e4bcc0a9c"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:816568614ecb22b18a010c7a12559c19f6fe993526af88e95a76d5a60b8b75fb"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c6564c0947a7f52e4792983f8e6cf9bac140438ebf81f527a21d944f2fd0a40"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c4a128527fe415d73cf1f70a9a688d06130d5810be69f3b553bf7b45e8acf79"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a49e1d7a4978ed554f095430b89ecc23f42014a50ac385eb0c4d163ce213c325"},
+ {file = "rpds_py-0.25.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d74ec9bc0e2feb81d3f16946b005748119c0f52a153f6db6a29e8cd68636f295"},
+ {file = "rpds_py-0.25.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3af5b4cc10fa41e5bc64e5c198a1b2d2864337f8fcbb9a67e747e34002ce812b"},
+ {file = "rpds_py-0.25.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:79dc317a5f1c51fd9c6a0c4f48209c6b8526d0524a6904fc1076476e79b00f98"},
+ {file = "rpds_py-0.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1521031351865e0181bc585147624d66b3b00a84109b57fcb7a779c3ec3772cd"},
+ {file = "rpds_py-0.25.1-cp313-cp313-win32.whl", hash = "sha256:5d473be2b13600b93a5675d78f59e63b51b1ba2d0476893415dfbb5477e65b31"},
+ {file = "rpds_py-0.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:a7b74e92a3b212390bdce1d93da9f6488c3878c1d434c5e751cbc202c5e09500"},
+ {file = "rpds_py-0.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:dd326a81afe332ede08eb39ab75b301d5676802cdffd3a8f287a5f0b694dc3f5"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:a58d1ed49a94d4183483a3ce0af22f20318d4a1434acee255d683ad90bf78129"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f251bf23deb8332823aef1da169d5d89fa84c89f67bdfb566c49dea1fccfd50d"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dbd586bfa270c1103ece2109314dd423df1fa3d9719928b5d09e4840cec0d72"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6d273f136e912aa101a9274c3145dcbddbe4bac560e77e6d5b3c9f6e0ed06d34"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:666fa7b1bd0a3810a7f18f6d3a25ccd8866291fbbc3c9b912b917a6715874bb9"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:921954d7fbf3fccc7de8f717799304b14b6d9a45bbeec5a8d7408ccbf531faf5"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d86373ff19ca0441ebeb696ef64cb58b8b5cbacffcda5a0ec2f3911732a194"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c8980cde3bb8575e7c956a530f2c217c1d6aac453474bf3ea0f9c89868b531b6"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:8eb8c84ecea987a2523e057c0d950bcb3f789696c0499290b8d7b3107a719d78"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:e43a005671a9ed5a650f3bc39e4dbccd6d4326b24fb5ea8be5f3a43a6f576c72"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58f77c60956501a4a627749a6dcb78dac522f249dd96b5c9f1c6af29bfacfb66"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-win32.whl", hash = "sha256:2cb9e5b5e26fc02c8a4345048cd9998c2aca7c2712bd1b36da0c72ee969a3523"},
+ {file = "rpds_py-0.25.1-cp313-cp313t-win_amd64.whl", hash = "sha256:401ca1c4a20cc0510d3435d89c069fe0a9ae2ee6495135ac46bdd49ec0495763"},
+ {file = "rpds_py-0.25.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ce4c8e485a3c59593f1a6f683cf0ea5ab1c1dc94d11eea5619e4fb5228b40fbd"},
+ {file = "rpds_py-0.25.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8222acdb51a22929c3b2ddb236b69c59c72af4019d2cba961e2f9add9b6e634"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4593c4eae9b27d22df41cde518b4b9e4464d139e4322e2127daa9b5b981b76be"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd035756830c712b64725a76327ce80e82ed12ebab361d3a1cdc0f51ea21acb0"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:114a07e85f32b125404f28f2ed0ba431685151c037a26032b213c882f26eb908"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dec21e02e6cc932538b5203d3a8bd6aa1480c98c4914cb88eea064ecdbc6396a"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09eab132f41bf792c7a0ea1578e55df3f3e7f61888e340779b06050a9a3f16e9"},
+ {file = "rpds_py-0.25.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c98f126c4fc697b84c423e387337d5b07e4a61e9feac494362a59fd7a2d9ed80"},
+ {file = "rpds_py-0.25.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0e6a327af8ebf6baba1c10fadd04964c1965d375d318f4435d5f3f9651550f4a"},
+ {file = "rpds_py-0.25.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:bc120d1132cff853ff617754196d0ac0ae63befe7c8498bd67731ba368abe451"},
+ {file = "rpds_py-0.25.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:140f61d9bed7839446bdd44852e30195c8e520f81329b4201ceead4d64eb3a9f"},
+ {file = "rpds_py-0.25.1-cp39-cp39-win32.whl", hash = "sha256:9c006f3aadeda131b438c3092124bd196b66312f0caa5823ef09585a669cf449"},
+ {file = "rpds_py-0.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:a61d0b2c7c9a0ae45732a77844917b427ff16ad5464b4d4f5e4adb955f582890"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b24bf3cd93d5b6ecfbedec73b15f143596c88ee249fa98cefa9a9dc9d92c6f28"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:0eb90e94f43e5085623932b68840b6f379f26db7b5c2e6bcef3179bd83c9330f"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d50e4864498a9ab639d6d8854b25e80642bd362ff104312d9770b05d66e5fb13"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c9409b47ba0650544b0bb3c188243b83654dfe55dcc173a86832314e1a6a35d"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:796ad874c89127c91970652a4ee8b00d56368b7e00d3477f4415fe78164c8000"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85608eb70a659bf4c1142b2781083d4b7c0c4e2c90eff11856a9754e965b2540"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4feb9211d15d9160bc85fa72fed46432cdc143eb9cf6d5ca377335a921ac37b"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ccfa689b9246c48947d31dd9d8b16d89a0ecc8e0e26ea5253068efb6c542b76e"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:3c5b317ecbd8226887994852e85de562f7177add602514d4ac40f87de3ae45a8"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:454601988aab2c6e8fd49e7634c65476b2b919647626208e376afcd22019eeb8"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:1c0c434a53714358532d13539272db75a5ed9df75a4a090a753ac7173ec14e11"},
+ {file = "rpds_py-0.25.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f73ce1512e04fbe2bc97836e89830d6b4314c171587a99688082d090f934d20a"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ee86d81551ec68a5c25373c5643d343150cc54672b5e9a0cafc93c1870a53954"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:89c24300cd4a8e4a51e55c31a8ff3918e6651b241ee8876a42cc2b2a078533ba"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:771c16060ff4e79584dc48902a91ba79fd93eade3aa3a12d6d2a4aadaf7d542b"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:785ffacd0ee61c3e60bdfde93baa6d7c10d86f15655bd706c89da08068dc5038"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a40046a529cc15cef88ac5ab589f83f739e2d332cb4d7399072242400ed68c9"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85fc223d9c76cabe5d0bff82214459189720dc135db45f9f66aa7cffbf9ff6c1"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0be9965f93c222fb9b4cc254235b3b2b215796c03ef5ee64f995b1b69af0762"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8378fa4a940f3fb509c081e06cb7f7f2adae8cf46ef258b0e0ed7519facd573e"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:33358883a4490287e67a2c391dfaea4d9359860281db3292b6886bf0be3d8692"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1d1fadd539298e70cac2f2cb36f5b8a65f742b9b9f1014dd4ea1f7785e2470bf"},
+ {file = "rpds_py-0.25.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a46c2fb2545e21181445515960006e85d22025bd2fe6db23e76daec6eb689fe"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:50f2c501a89c9a5f4e454b126193c5495b9fb441a75b298c60591d8a2eb92e1b"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d779b325cc8238227c47fbc53964c8cc9a941d5dbae87aa007a1f08f2f77b23"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:036ded36bedb727beeabc16dc1dad7cb154b3fa444e936a03b67a86dc6a5066e"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:245550f5a1ac98504147cba96ffec8fabc22b610742e9150138e5d60774686d7"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff7c23ba0a88cb7b104281a99476cccadf29de2a0ef5ce864959a52675b1ca83"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e37caa8cdb3b7cf24786451a0bdb853f6347b8b92005eeb64225ae1db54d1c2b"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2f48ab00181600ee266a095fe815134eb456163f7d6699f525dee471f312cf"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e5fc7484fa7dce57e25063b0ec9638ff02a908304f861d81ea49273e43838c1"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:d3c10228d6cf6fe2b63d2e7985e94f6916fa46940df46b70449e9ff9297bd3d1"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:5d9e40f32745db28c1ef7aad23f6fc458dc1e29945bd6781060f0d15628b8ddf"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:35a8d1a24b5936b35c5003313bc177403d8bdef0f8b24f28b1c4a255f94ea992"},
+ {file = "rpds_py-0.25.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6099263f526efff9cf3883dfef505518730f7a7a93049b1d90d42e50a22b4793"},
+ {file = "rpds_py-0.25.1.tar.gz", hash = "sha256:8960b6dac09b62dac26e75d7e2c4a22efb835d827a7278c34f72b2b84fa160e3"},
+]
+
[[package]]
name = "s3fs"
version = "2025.5.1"
@@ -3411,4 +3629,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.12,<3.15"
-content-hash = "4a0aaa0b0d1d756a4d7e8b1248a1ca8dfe68810a3b525950d338eb3fe2524121"
+content-hash = "b52848d8ed7c43fa58ba8a240fe8008dcc8bf5f00dc7a8c8b22f3453289be683"
diff --git a/src/worker-base/pyproject.toml b/src/worker-base/pyproject.toml
index 2496cc2..c555d6a 100644
--- a/src/worker-base/pyproject.toml
+++ b/src/worker-base/pyproject.toml
@@ -7,6 +7,7 @@ description = ""
python = ">=3.12,<3.15"
flint = { path = "../common-lib/flint", develop = false }
ipykernel = "^6.29.5"
+nbformat = "^5.10.4"
[build-system]
requires = ["poetry-core"]
diff --git a/src/worker-base/startup/entrypoint.sh b/src/worker-base/startup/entrypoint.sh
new file mode 100644
index 0000000..caf9786
--- /dev/null
+++ b/src/worker-base/startup/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+
+# # Configure S3FS
+echo ${STORAGE_USER}:${STORAGE_PASSWORD} > /root/.passwd-s3fs
+chmod 600 /root/.passwd-s3fs
+s3fs metastore /mnt/metastore \
+ -o url="${STORAGE_ENDPOINT}" \
+ -o use_path_request_style \
+ -o allow_other \
+ -o passwd_file=/root/.passwd-s3fs \
+ -o uid="$(id -u)" \
+ -o gid="$(id -g)" \
+ -o nonempty \
+ -o curldbg -o dbglevel=info \
+ -o logfile=/var/log/s3fs-metastore.log
+mkdir -p /mnt/metastore/workspace
+
+cd /mnt/metastore/workspace
+python -m ipykernel_launcher -f /tmp/connection.json &
+KERNEL_PID=$!
+
+trap 'kill -TERM $KERNEL_PID 2>/dev/null' TERM INT
+
+exec python /root/watchdog.py --kernel-pid "$KERNEL_PID"
\ No newline at end of file
diff --git a/src/workspace/Dockerfile b/src/workspace/Dockerfile
index b226e8b..929dc66 100644
--- a/src/workspace/Dockerfile
+++ b/src/workspace/Dockerfile
@@ -69,7 +69,12 @@ COPY --from=builder /usr/local/etc/jupyter /usr/local/etc/jupyter
COPY --from=builder /app/workspace /app/workspace
COPY --from=builder /usr/local/bin /usr/local/bin
-RUN mkdir -p /srv/workspace
+# Configure S3FS
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ s3fs fuse libfuse2 && \
+ rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /mnt/metastore
COPY --from=builder /app/workspace/entrypoint.sh /root/entrypoint.sh
RUN chmod +x /root/entrypoint.sh
diff --git a/src/workspace/backend/jupyter_notebook_config.py b/src/workspace/backend/jupyter_notebook_config.py
index da65364..5e6e54c 100644
--- a/src/workspace/backend/jupyter_notebook_config.py
+++ b/src/workspace/backend/jupyter_notebook_config.py
@@ -46,4 +46,4 @@ def set_doc_manager_settings():
c.KernelSpecManager.ensure_native_kernel = False
-c.ServerApp.root_dir = "/srv/workspace"
\ No newline at end of file
+c.ServerApp.root_dir = "/mnt/metastore/workspace"
\ No newline at end of file
diff --git a/src/workspace/entrypoint.sh b/src/workspace/entrypoint.sh
index 47e05eb..6807f1b 100644
--- a/src/workspace/entrypoint.sh
+++ b/src/workspace/entrypoint.sh
@@ -1,7 +1,22 @@
#!/bin/bash
set -e
-echo "🚀 Launching Jupyter..."
+# Configure S3FS
+echo ${STORAGE_USER}:${STORAGE_PASSWORD} > /root/.passwd-s3fs
+chmod 600 /root/.passwd-s3fs
+
+s3fs metastore /mnt/metastore \
+ -o url=http://storage:8000 \
+ -o use_path_request_style \
+ -o allow_other \
+ -o passwd_file=/root/.passwd-s3fs \
+ -o uid="$(id -u)" \
+ -o gid="$(id -g)" \
+ -o nonempty \
+ -o curldbg -o dbglevel=info \
+ -o logfile=/var/log/s3fs-metastore.log
+mkdir -p /mnt/metastore/workspace
+
exec python -m jupyter_server \
--config=/root/.jupyter/jupyter_notebook_config.py \
--ip=0.0.0.0 \