From 92f20f3a781f9a36c3f99dfbc16ad6da1f5b7725 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Wed, 18 Feb 2026 02:45:31 +0000 Subject: [PATCH 01/22] feat(infra): integrate openziti stack --- .gitignore | 1 + .npmrc | 1 + README.md | 22 +- docker-compose.yml | 97 +++++++ docs/containers/ziti.md | 93 +++++++ docs/product-spec.md | 2 + docs/technical-overview.md | 1 + packages/docker-runner/package.json | 5 + packages/docker-runner/src/service/config.ts | 25 ++ packages/docker-runner/src/service/main.ts | 24 +- .../docker-runner/src/service/ziti.ingress.ts | 65 +++++ .../docker-runner/src/types/openziti.d.ts | 8 + packages/platform-server/.env.example | 14 + packages/platform-server/package.json | 3 + .../src/core/services/config.service.ts | 137 ++++++++++ .../platform-server/src/infra/infra.module.ts | 18 +- .../src/infra/ziti/ziti.bootstrap.service.ts | 43 +++ .../src/infra/ziti/ziti.identity.manager.ts | 70 +++++ .../src/infra/ziti/ziti.management.client.ts | 234 +++++++++++++++++ .../src/infra/ziti/ziti.reconciler.ts | 247 ++++++++++++++++++ .../infra/ziti/ziti.runnerProxy.service.ts | 123 +++++++++ .../src/infra/ziti/ziti.types.ts | 75 ++++++ .../platform-server/src/types/openziti.d.ts | 5 + 23 files changed, 1307 insertions(+), 6 deletions(-) create mode 100644 .npmrc create mode 100644 docs/containers/ziti.md create mode 100644 packages/docker-runner/src/service/ziti.ingress.ts create mode 100644 packages/docker-runner/src/types/openziti.d.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.identity.manager.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.management.client.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.reconciler.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts create mode 100644 packages/platform-server/src/infra/ziti/ziti.types.ts create mode 100644 packages/platform-server/src/types/openziti.d.ts diff --git a/.gitignore b/.gitignore index 49e4d87f7..16674a733 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules .turbo node-compile-cache/ .env +.ziti/ packages/platform-server/build-ts # Bundled artifacts generated during build packages/platform-server/dist diff --git a/.npmrc b/.npmrc new file mode 100644 index 000000000..742311090 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +allow-scripts=@openziti/ziti-sdk-nodejs diff --git a/README.md b/README.md index 31fbb2349..a021e946c 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ pnpm install ```bash docker compose up -d # Starts postgres (5442), agents-db (5443), vault (8200), ncps (8501), -# litellm (127.0.0.1:4000), docker-runner (7071) +# litellm (127.0.0.1:4000), docker-runner (7071), and the OpenZiti controller stack # Optional monitoring (prometheus/grafana) lives in docker-compose.monitoring.yml. # Enable with: docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d ``` @@ -165,6 +165,20 @@ docker run --rm -p 8080:80 \ ghcr.io/agynio/platform-ui:latest ``` +### Secure docker-runner connectivity (OpenZiti) + +The dev stack now ships an OpenZiti controller, initializer, and edge router. To route docker-runner traffic through the +overlay instead of the Docker bridge network: + +1. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). +2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and + `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). +3. Start the controller stack: `docker compose up -d ziti-controller controller-init ziti-edge-router`. +4. Launch docker-runner and platform-server normally. The server will reconcile the controller, enroll identities, and + expose a local proxy on `127.0.0.1:17071` for all docker-runner calls. + +See [docs/containers/ziti.md](docs/containers/ziti.md) for the full walkthrough and smoke test commands. + ## Configuration Key environment variables (server) from packages/platform-server/.env.example and src/core/services/config.service.ts: @@ -188,6 +202,12 @@ Key environment variables (server) from packages/platform-server/.env.example an - DOCKER_RUNNER_BASE_URL (required; default http://docker-runner:7071) - DOCKER_RUNNER_SHARED_SECRET (required HMAC credential) - DOCKER_RUNNER_TIMEOUT_MS (optional request timeout; default 30000) +- OpenZiti (optional secure docker-runner tunnel): + - ZITI_ENABLED (default false) — enable controller reconciliation + proxy + - ZITI_MANAGEMENT_URL / ZITI_USERNAME / ZITI_PASSWORD — controller credentials + - ZITI_SERVICE_NAME / ZITI_ROUTER_NAME — service plus edge router handles + - ZITI_PLATFORM_IDENTITY_FILE / ZITI_RUNNER_IDENTITY_FILE — identity output paths under `./.ziti/` + - ZITI_RUNNER_PROXY_PORT (default 17071) — local HTTP proxy for docker-runner calls - Nix/NCPS: - NCPS_ENABLED (default false) - NCPS_URL_SERVER, NCPS_URL_CONTAINER (default http://ncps:8501) diff --git a/docker-compose.yml b/docker-compose.yml index 4f438a897..85f83390c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -247,18 +247,115 @@ services: networks: - agents_net + ziti-controller: + image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + container_name: ziti-controller + restart: unless-stopped + oom_score_adj: -900 + entrypoint: + - "/var/openziti/scripts/run-controller.sh" + environment: + ZITI_CTRL_NAME: ${ZITI_CTRL_NAME:-dev-controller} + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} + ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} + ZITI_CTRL_EDGE_IP_OVERRIDE: ${ZITI_CTRL_EDGE_IP_OVERRIDE:-127.0.0.1} + ZITI_CTRL_ADVERTISED_PORT: ${ZITI_CTRL_ADVERTISED_PORT:-6262} + ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION: ${ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION:-168h} + ZITI_ROUTER_ENROLLMENT_DURATION: ${ZITI_ROUTER_ENROLLMENT_DURATION:-168h} + ZITI_USER: ${ZITI_USER:-admin} + ZITI_PWD: ${ZITI_PWD:-admin} + ports: + - "127.0.0.1:${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280}:${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280}" + - "127.0.0.1:${ZITI_CTRL_ADVERTISED_PORT:-6262}:${ZITI_CTRL_ADVERTISED_PORT:-6262}" + healthcheck: + test: + [ + "CMD-SHELL", + "curl -m 1 -s -k -f https://${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller}:${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280}/edge/client/v1/version", + ] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - type: bind + source: ./.ziti/controller + target: /persistent + networks: + - agents_net + + ziti-controller-init: + image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + container_name: ziti-controller-init + restart: "no" + depends_on: + ziti-controller: + condition: service_healthy + entrypoint: + - "/var/openziti/scripts/run-with-ziti-cli.sh" + command: + - "/var/openziti/scripts/access-control.sh" + environment: + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} + ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} + volumes: + - type: bind + source: ./.ziti/controller + target: /persistent + networks: + - agents_net + + ziti-edge-router: + image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + container_name: ziti-edge-router + restart: unless-stopped + depends_on: + ziti-controller-init: + condition: service_completed_successfully + entrypoint: /bin/bash + command: "/var/openziti/scripts/run-router.sh edge" + environment: + ZITI_CTRL_ADVERTISED_ADDRESS: ${ZITI_CTRL_ADVERTISED_ADDRESS:-ziti-controller} + ZITI_CTRL_ADVERTISED_PORT: ${ZITI_CTRL_ADVERTISED_PORT:-6262} + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} + ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} + ZITI_ROUTER_NAME: ${ZITI_ROUTER_NAME:-dev-edge-router} + ZITI_ROUTER_ADVERTISED_ADDRESS: ${ZITI_ROUTER_ADVERTISED_ADDRESS:-ziti-edge-router} + ZITI_ROUTER_PORT: ${ZITI_ROUTER_PORT:-3022} + ZITI_ROUTER_LISTENER_BIND_PORT: ${ZITI_ROUTER_LISTENER_BIND_PORT:-10080} + ZITI_ROUTER_ROLES: ${ZITI_ROUTER_ROLES:-public} + ZITI_USER: ${ZITI_USER:-admin} + ZITI_PWD: ${ZITI_PWD:-admin} + ports: + - "127.0.0.1:${ZITI_ROUTER_PORT:-3022}:${ZITI_ROUTER_PORT:-3022}" + volumes: + - type: bind + source: ./.ziti/controller + target: /persistent + networks: + - agents_net + docker-runner: build: context: . dockerfile: packages/docker-runner/Dockerfile restart: unless-stopped + depends_on: + ziti-edge-router: + condition: service_started environment: DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} DOCKER_RUNNER_PORT: ${DOCKER_RUNNER_PORT:-7071} + ZITI_ENABLED: ${ZITI_ENABLED:-false} + ZITI_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} + ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} volumes: - type: bind source: /var/run/docker.sock target: /var/run/docker.sock + - type: bind + source: ./.ziti/identities + target: /opt/app/.ziti/identities + read_only: true ports: - "${DOCKER_RUNNER_PORT:-7071}:7071" networks: diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md new file mode 100644 index 000000000..d0f92ec9f --- /dev/null +++ b/docs/containers/ziti.md @@ -0,0 +1,93 @@ +# OpenZiti integration + +The local development stack now provisions an OpenZiti controller, initializer, and edge router. The platform-server +reconciles controller state at startup (service, policies, and identities) and stores identity material under +`./.ziti/identities`. A lightweight local HTTP proxy (`127.0.0.1:17071`) tunnels docker-runner traffic through the +OpenZiti overlay instead of the Docker bridge network when enabled. + +## Prerequisites + +1. Install dependencies and explicitly allow the OpenZiti SDK build step (pnpm blocks install scripts by default): + +```bash +pnpm approve-builds +# Select @openziti/ziti-sdk-nodejs and confirm +``` + +> If interactive approvals are not available you can run the install script once manually: +> +> ```bash +> pnpm --dir node_modules/.pnpm/@openziti+ziti-sdk-nodejs@0.27.0/node_modules/@openziti/ziti-sdk-nodejs run install +> ``` + +2. Ensure the dev stack is running: + +```bash +docker compose up -d ziti-controller controller-init ziti-edge-router +``` + +3. Copy `.env` files and enable OpenZiti flags: + +- `packages/platform-server/.env` + +``` +ZITI_ENABLED=true +ZITI_MANAGEMENT_URL=https://ziti-controller:1280/edge/management/v1 +ZITI_USERNAME=admin +ZITI_PASSWORD=admin +ZITI_INSECURE_TLS=true +ZITI_SERVICE_NAME=dev.agyn-platform.platform-api +ZITI_ROUTER_NAME=dev-edge-router +ZITI_PLATFORM_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json +``` + +- `packages/docker-runner/.env` (or container env) + +``` +ZITI_ENABLED=true +ZITI_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_SERVICE_NAME=dev.agyn-platform.platform-api +``` + +The docker-compose service already mounts `./.ziti` into both platform-server and docker-runner containers. Local +development outside Docker can re-use the same paths. + +## Runtime flow + +1. Platform-server bootstraps the controller via the Ziti Management API: + - Creates/updates the service (`dev.agyn-platform.platform-api`). + - Ensures bind/dial service policies and a service-edge-router policy targeting `dev-edge-router`. + - Creates device identities for the server (`component.platform-server`) and docker-runner (`component.docker-runner`). + - Generates OTT enrollments and writes identities to `.ziti/identities/`. +2. Ziti runner proxy starts on `127.0.0.1:17071` and dials the service using the platform-server identity. All requests to + docker-runner are routed through this proxy when `ZITI_ENABLED=true`. +3. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an + OpenZiti Express listener that proxies traffic to the local Fastify server. + +## Smoke test + +After both services start with `ZITI_ENABLED=true`: + +1. Verify the local proxy is healthy (platform-server side): + +```bash +curl http://127.0.0.1:17071/v1/ready +``` + +Expected response: + +```json +{"status":"ready"} +``` + +2. Confirm docker-runner bridged the OpenZiti ingress: + +```bash +docker logs docker-runner | grep "Ziti ingress ready" +``` + +Seeing the readiness log after step 1 indicates the end-to-end tunnel is operational. + +> To reset the environment delete `./.ziti/identities` and `./.ziti/tmp`, then restart the stack so the platform-server +> can re-enroll identities. diff --git a/docs/product-spec.md b/docs/product-spec.md index 023fcd8bb..cd481f8b8 100644 --- a/docs/product-spec.md +++ b/docs/product-spec.md @@ -117,6 +117,7 @@ Configuration matrix (server env vars) - VAULT_ADDR, VAULT_TOKEN - DOCKER_MIRROR_URL (default http://registry-mirror:5000) - DOCKER_RUNNER_BASE_URL, DOCKER_RUNNER_SHARED_SECRET (required for docker-runner), plus optional DOCKER_RUNNER_TIMEOUT_MS (default 30000). + - ZITI_* (see docs/containers/ziti.md) to route docker-runner traffic through the OpenZiti overlay. - MCP_TOOLS_STALE_TIMEOUT_MS - LANGGRAPH_CHECKPOINTER: postgres (default) - POSTGRES_URL (postgres connection string) @@ -133,6 +134,7 @@ Runbooks - Local dev - Prereqs: Node 18+, pnpm, Docker, Postgres. - Set: LLM_PROVIDER=litellm, LITELLM_BASE_URL, LITELLM_MASTER_KEY, GITHUB_*, GH_TOKEN, AGENTS_DATABASE_URL, DOCKER_RUNNER_BASE_URL, DOCKER_RUNNER_SHARED_SECRET. Optional VAULT_* and DOCKER_MIRROR_URL. + - Optional secure runner: enable ZITI_* env vars on both platform-server and docker-runner; ensure compose Ziti services are up. - Start deps (compose or local Postgres) - Server: pnpm -w -F @agyn/platform-server dev - UI: pnpm -w -F @agyn/platform-ui dev diff --git a/docs/technical-overview.md b/docs/technical-overview.md index e9a7ac0fa..a9222f55f 100644 --- a/docs/technical-overview.md +++ b/docs/technical-overview.md @@ -70,6 +70,7 @@ Remote Docker runner - The platform-server always routes container lifecycle, exec, and log streaming calls through the `@agyn/docker-runner` service. - The runner exposes authenticated Fastify HTTP/SSE/WebSocket endpoints with HMAC headers derived solely from `DOCKER_RUNNER_SHARED_SECRET`. - Only the docker-runner service mounts `/var/run/docker.sock` in default stacks; platform-server and auxiliary services talk to it over the internal network (default http://docker-runner:7071). +- When `ZITI_ENABLED=true`, platform-server launches a local proxy on `127.0.0.1:17071` backed by the OpenZiti Node SDK and docker-runner binds the same API to the OpenZiti service (`dev.agyn-platform.platform-api`). - Container events are forwarded via SSE so the existing watcher pipeline (ContainerEventProcessor, cleanup jobs, metrics) remains unchanged. Defaults and toggles diff --git a/packages/docker-runner/package.json b/packages/docker-runner/package.json index 941443a00..3c07311dc 100644 --- a/packages/docker-runner/package.json +++ b/packages/docker-runner/package.json @@ -14,9 +14,12 @@ "dependencies": { "@fastify/websocket": "^11.2.0", "@nestjs/common": "^11.1.7", + "@openziti/ziti-sdk-nodejs": "^0.27.0", "dockerode": "^4.0.8", "dotenv": "^17.2.2", "fastify": "^5.6.1", + "express": "^4.21.1", + "http-proxy": "^1.18.1", "node-fetch-native": "^1.6.7", "pino": "^10.1.0", "undici": "^6.19.8", @@ -25,6 +28,8 @@ }, "devDependencies": { "@types/dockerode": "^3.3.44", + "@types/express": "^4.17.21", + "@types/http-proxy": "^1.17.14", "@types/node": "^24.5.1", "@types/ws": "^8.5.11", "eslint": "^9.13.0", diff --git a/packages/docker-runner/src/service/config.ts b/packages/docker-runner/src/service/config.ts index fc84bfa86..642871f35 100644 --- a/packages/docker-runner/src/service/config.ts +++ b/packages/docker-runner/src/service/config.ts @@ -1,5 +1,18 @@ import { z } from 'zod'; +const booleanFlag = (defaultValue: boolean) => + z + .union([z.boolean(), z.string()]) + .default(defaultValue ? 'true' : 'false') + .transform((value) => { + if (typeof value === 'boolean') return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return defaultValue; + if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true; + if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false; + return defaultValue; + }); + const runnerConfigSchema = z.object({ port: z .union([z.string(), z.number()]) @@ -19,6 +32,13 @@ const runnerConfigSchema = z.object({ }), dockerSocket: z.string().default('/var/run/docker.sock'), logLevel: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), + ziti: z + .object({ + enabled: booleanFlag(false), + identityFile: z.string().default('.ziti/identities/dev.agyn-platform.docker-runner.json'), + serviceName: z.string().default('dev.agyn-platform.platform-api'), + }) + .default({}), }); export type RunnerConfig = z.infer; @@ -31,6 +51,11 @@ export function loadRunnerConfig(env: NodeJS.ProcessEnv = process.env): RunnerCo signatureTtlMs: env.DOCKER_RUNNER_SIGNATURE_TTL_MS, dockerSocket: env.DOCKER_SOCKET ?? env.DOCKER_RUNNER_SOCKET, logLevel: env.DOCKER_RUNNER_LOG_LEVEL, + ziti: { + enabled: env.ZITI_ENABLED, + identityFile: env.ZITI_IDENTITY_FILE, + serviceName: env.ZITI_SERVICE_NAME, + }, }); if (!parsed.success) { throw new Error(`Invalid docker-runner configuration: ${parsed.error.message}`); diff --git a/packages/docker-runner/src/service/main.ts b/packages/docker-runner/src/service/main.ts index 61be2f7f2..03e6462d9 100644 --- a/packages/docker-runner/src/service/main.ts +++ b/packages/docker-runner/src/service/main.ts @@ -1,15 +1,37 @@ import './env'; +import type { FastifyInstance } from 'fastify'; + import { loadRunnerConfig } from './config'; import { createRunnerApp } from './app'; +import { startZitiIngress } from './ziti.ingress'; async function bootstrap(): Promise { + let app: FastifyInstance | undefined; + let closeZiti: (() => Promise) | undefined; + try { const config = loadRunnerConfig(); - const app = createRunnerApp(config); + app = createRunnerApp(config); await app.listen({ port: config.port, host: config.host }); + const ingress = await startZitiIngress(config); + closeZiti = ingress?.close; + + const shutdown = async () => { + await closeZiti?.(); + if (app) { + await app.close(); + } + process.exit(0); + }; + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); } catch (error) { console.error('docker-runner failed to start', error); + await closeZiti?.(); + if (app) { + await app.close(); + } process.exit(1); } } diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts new file mode 100644 index 000000000..3e3672e53 --- /dev/null +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -0,0 +1,65 @@ +import { type Server as HttpServer } from 'node:http'; +import type { Duplex } from 'node:stream'; + +import express, { type Express } from 'express'; +import httpProxy from 'http-proxy'; + +import type { RunnerConfig } from './config'; + +type ZitiIngressHandle = { + close: () => Promise; +}; + +export async function startZitiIngress(config: RunnerConfig): Promise { + if (!config.ziti.enabled) { + return undefined; + } + + const ziti = await import('@openziti/ziti-sdk-nodejs'); + await ziti.init(config.ziti.identityFile); + + const app = ziti.express(express, config.ziti.serviceName); + const target = `http://${config.host}:${config.port}`; + const proxy = httpProxy.createProxyServer({ + target, + changeOrigin: true, + ws: true, + }); + + app.use((req, res) => { + proxy.web(req, res, undefined, (error) => { + const message = error instanceof Error ? error.message : 'proxy_failed'; + res.status(502).json({ error: 'ziti_ingress_error', message }); + }); + }); + + const server = await listenAsync(app); + server.on('upgrade', (req, socket: Duplex, head) => { + proxy.ws(req, socket, head, undefined, () => socket.destroy()); + }); + + // eslint-disable-next-line no-console -- CLI startup log + console.info(`Ziti ingress ready for service ${config.ziti.serviceName}`); + + return { + close: async () => { + proxy.close(); + await closeServer(server); + }, + }; +} + +const listenAsync = (app: Express): Promise => + new Promise((resolve, reject) => { + const handleError = (error: unknown) => reject(error instanceof Error ? error : new Error('ziti ingress failed')); + const server = app.listen(() => { + server.off('error', handleError); + resolve(server); + }); + server.once('error', handleError); + }); + +const closeServer = (server: HttpServer): Promise => + new Promise((resolve) => { + server.close(() => resolve()); + }); diff --git a/packages/docker-runner/src/types/openziti.d.ts b/packages/docker-runner/src/types/openziti.d.ts new file mode 100644 index 000000000..e99b1fe0a --- /dev/null +++ b/packages/docker-runner/src/types/openziti.d.ts @@ -0,0 +1,8 @@ +declare module '@openziti/ziti-sdk-nodejs' { + import type { Express } from 'express'; + + export function init(identityPath: string): Promise; + export function httpAgent(): unknown; + export function enroll(jwtPath: string): Promise; + export function express(appFactory: typeof import('express'), serviceName: string): Express; +} diff --git a/packages/platform-server/.env.example b/packages/platform-server/.env.example index eaa936249..0013d0433 100644 --- a/packages/platform-server/.env.example +++ b/packages/platform-server/.env.example @@ -61,3 +61,17 @@ DOCKER_RUNNER_SHARED_SECRET=dev-shared-secret # Container retention window (in days). Set to 0 to retain indefinitely. CONTAINERS_RETENTION_DAYS=14 + +# OpenZiti integration (disabled by default) +ZITI_ENABLED=false +ZITI_MANAGEMENT_URL=https://127.0.0.1:1280/edge/management/v1 +ZITI_USERNAME=admin +ZITI_PASSWORD=admin +ZITI_INSECURE_TLS=true +ZITI_SERVICE_NAME=dev.agyn-platform.platform-api +ZITI_ROUTER_NAME=dev-edge-router +ZITI_RUNNER_PROXY_PORT=17071 +ZITI_PLATFORM_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json +# Enrollment JWT lifetime (seconds) +ZITI_ENROLLMENT_TTL_SECONDS=900 diff --git a/packages/platform-server/package.json b/packages/platform-server/package.json index 669058e5e..eabc08aad 100644 --- a/packages/platform-server/package.json +++ b/packages/platform-server/package.json @@ -33,6 +33,7 @@ "@langchain/langgraph-checkpoint-postgres": "1.0.0", "@langchain/openai": "1.0.0", "@modelcontextprotocol/sdk": "^1.18.1", + "@openziti/ziti-sdk-nodejs": "^0.27.0", "@nestjs/common": "^11.1.7", "@nestjs/core": "^11.1.7", "@nestjs/platform-fastify": "^11.1.7", @@ -50,6 +51,7 @@ "iconv-lite": "^0.6.3", "lodash-es": "^4.17.21", "md5": "^2.3.0", + "http-proxy": "^1.18.1", "mustache": "^4.2.0", "nestjs-pino": "^4.5.0", "node-fetch-native": "^1.6.7", @@ -73,6 +75,7 @@ "@langchain/langgraph-cli": "0.0.66", "@nestjs/testing": "^11.1.8", "@types/json-schema": "^7.0.15", + "@types/http-proxy": "^1.17.14", "@types/lodash-es": "^4.17.12", "@types/md5": "^2.3.5", "@types/node": "^24.5.1", diff --git a/packages/platform-server/src/core/services/config.service.ts b/packages/platform-server/src/core/services/config.service.ts index d6080b05e..306504217 100644 --- a/packages/platform-server/src/core/services/config.service.ts +++ b/packages/platform-server/src/core/services/config.service.ts @@ -3,6 +3,31 @@ import * as dotenv from 'dotenv'; import { z } from 'zod'; dotenv.config(); +const booleanFlag = (defaultValue: boolean) => + z + .union([z.boolean(), z.string()]) + .default(defaultValue ? 'true' : 'false') + .transform((value) => { + if (typeof value === 'boolean') return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return defaultValue; + if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true; + if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false; + return defaultValue; + }); + +const numberFlag = (defaultValue: number) => + z + .union([z.string(), z.number()]) + .default(String(defaultValue)) + .transform((value) => { + if (typeof value === 'number') return Number.isFinite(value) ? value : defaultValue; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : defaultValue; + }); + +const trimUrl = (value: string): string => value.trim().replace(/\/+$/, ''); + export const configSchema = z.object({ // GitHub settings are optional to allow dev boot without GitHub githubAppId: z.string().min(1).optional(), @@ -188,6 +213,33 @@ export const configSchema = z.object({ .map((x) => x.trim()) .filter((x) => !!x), ), + ziti: z + .object({ + enabled: booleanFlag(false), + managementUrl: z + .string() + .default('https://127.0.0.1:1280/edge/management/v1') + .transform((value) => trimUrl(value)), + username: z.string().default('admin'), + password: z.string().default('admin'), + insecureTls: booleanFlag(true), + serviceName: z.string().default('dev.agyn-platform.platform-api'), + routerName: z.string().default('dev-edge-router'), + runnerProxyHost: z.string().default('127.0.0.1'), + runnerProxyPort: numberFlag(17071), + platformIdentityName: z.string().default('dev.agyn-platform.platform-server'), + platformIdentityFile: z + .string() + .default('.ziti/identities/dev.agyn-platform.platform-server.json'), + runnerIdentityName: z.string().default('dev.agyn-platform.docker-runner'), + runnerIdentityFile: z + .string() + .default('.ziti/identities/dev.agyn-platform.docker-runner.json'), + identitiesDir: z.string().default('.ziti/identities'), + tmpDir: z.string().default('.ziti/tmp'), + enrollmentTtlSeconds: numberFlag(900), + }) + .default({}), }); export type Config = z.infer; @@ -326,6 +378,11 @@ export class ConfigService implements Config { } getDockerRunnerBaseUrl(): string { + if (this.isZitiEnabled()) { + const host = this.getZitiRunnerProxyHost(); + const port = this.getZitiRunnerProxyPort(); + return `http://${host}:${port}`; + } return this.dockerRunnerBaseUrl; } @@ -417,6 +474,68 @@ export class ConfigService implements Config { return this.params.nixRepoAllowlist ?? []; } + get zitiConfig(): Config['ziti'] { + return this.params.ziti; + } + + isZitiEnabled(): boolean { + return !!this.params.ziti?.enabled; + } + + getZitiManagementUrl(): string { + return this.params.ziti.managementUrl; + } + + getZitiCredentials(): { username: string; password: string } { + return { username: this.params.ziti.username, password: this.params.ziti.password }; + } + + getZitiInsecureTls(): boolean { + return this.params.ziti.insecureTls; + } + + getZitiServiceName(): string { + return this.params.ziti.serviceName; + } + + getZitiRouterName(): string { + return this.params.ziti.routerName; + } + + getZitiRunnerProxyHost(): string { + return this.params.ziti.runnerProxyHost; + } + + getZitiRunnerProxyPort(): number { + return this.params.ziti.runnerProxyPort; + } + + getZitiPlatformIdentity(): { name: string; file: string } { + return { + name: this.params.ziti.platformIdentityName, + file: this.params.ziti.platformIdentityFile, + }; + } + + getZitiRunnerIdentity(): { name: string; file: string } { + return { + name: this.params.ziti.runnerIdentityName, + file: this.params.ziti.runnerIdentityFile, + }; + } + + getZitiIdentityDirectory(): string { + return this.params.ziti.identitiesDir; + } + + getZitiTmpDirectory(): string { + return this.params.ziti.tmpDir; + } + + getZitiEnrollmentTtlSeconds(): number { + return this.params.ziti.enrollmentTtlSeconds; + } + // No global messaging adapter config in Slack-only v1 static fromEnv(): ConfigService { @@ -471,6 +590,24 @@ export class ConfigService implements Config { ncpsAuthToken: process.env.NCPS_AUTH_TOKEN, agentsDatabaseUrl: process.env.AGENTS_DATABASE_URL, corsOrigins: process.env.CORS_ORIGINS, + ziti: { + enabled: process.env.ZITI_ENABLED, + managementUrl: process.env.ZITI_MANAGEMENT_URL, + username: process.env.ZITI_USERNAME, + password: process.env.ZITI_PASSWORD, + insecureTls: process.env.ZITI_INSECURE_TLS, + serviceName: process.env.ZITI_SERVICE_NAME, + routerName: process.env.ZITI_ROUTER_NAME, + runnerProxyHost: process.env.ZITI_RUNNER_PROXY_HOST, + runnerProxyPort: process.env.ZITI_RUNNER_PROXY_PORT, + platformIdentityName: process.env.ZITI_PLATFORM_IDENTITY_NAME, + platformIdentityFile: process.env.ZITI_PLATFORM_IDENTITY_FILE, + runnerIdentityName: process.env.ZITI_RUNNER_IDENTITY_NAME, + runnerIdentityFile: process.env.ZITI_RUNNER_IDENTITY_FILE, + identitiesDir: process.env.ZITI_IDENTITIES_DIR, + tmpDir: process.env.ZITI_TMP_DIR, + enrollmentTtlSeconds: process.env.ZITI_ENROLLMENT_TTL_SECONDS, + }, }); const config = new ConfigService().init(parsed); ConfigService.register(config); diff --git a/packages/platform-server/src/infra/infra.module.ts b/packages/platform-server/src/infra/infra.module.ts index c43bd5233..27f5bfc20 100644 --- a/packages/platform-server/src/infra/infra.module.ts +++ b/packages/platform-server/src/infra/infra.module.ts @@ -25,11 +25,19 @@ import { DockerWorkspaceRuntimeProvider } from '../workspace/providers/docker.wo import { DOCKER_CLIENT, type DockerClient } from './container/dockerClient.token'; import { HttpDockerRunnerClient } from './container/httpDockerRunner.client'; import { DockerRunnerConnectivityProbe } from './container/dockerRunnerConnectivity.probe'; +import { ZitiIdentityManager } from './ziti/ziti.identity.manager'; +import { ZitiReconciler } from './ziti/ziti.reconciler'; +import { ZitiRunnerProxyService } from './ziti/ziti.runnerProxy.service'; +import { ZitiBootstrapService } from './ziti/ziti.bootstrap.service'; @Module({ imports: [CoreModule, VaultModule], providers: [ ArchiveService, + ZitiIdentityManager, + ZitiReconciler, + ZitiRunnerProxyService, + ZitiBootstrapService, { provide: ContainerRegistry, useFactory: async (prismaSvc: PrismaService) => { @@ -41,13 +49,15 @@ import { DockerRunnerConnectivityProbe } from './container/dockerRunnerConnectiv }, { provide: DOCKER_CLIENT, - useFactory: (config: ConfigService) => - new HttpDockerRunnerClient({ + useFactory: async (config: ConfigService, zitiBootstrap: ZitiBootstrapService) => { + await zitiBootstrap.ensureReady(); + return new HttpDockerRunnerClient({ baseUrl: config.getDockerRunnerBaseUrl(), sharedSecret: config.getDockerRunnerSharedSecret(), requestTimeoutMs: config.getDockerRunnerTimeoutMs(), - }), - inject: [ConfigService], + }); + }, + inject: [ConfigService, ZitiBootstrapService], }, DockerRunnerConnectivityProbe, { diff --git a/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts b/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts new file mode 100644 index 000000000..769bc76e6 --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts @@ -0,0 +1,43 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; + +import { ConfigService } from '../../core/services/config.service'; +import { ZitiReconciler } from './ziti.reconciler'; +import { ZitiRunnerProxyService } from './ziti.runnerProxy.service'; + +@Injectable() +export class ZitiBootstrapService implements OnModuleDestroy { + private readonly logger = new Logger(ZitiBootstrapService.name); + private initialization?: Promise; + + constructor( + private readonly config: ConfigService, + private readonly reconciler: ZitiReconciler, + private readonly proxy: ZitiRunnerProxyService, + ) {} + + ensureReady(): Promise { + if (!this.config?.isZitiEnabled()) { + return Promise.resolve(); + } + if (!this.initialization) { + this.initialization = this.initialize(); + } + return this.initialization; + } + + async onModuleDestroy(): Promise { + if (!this.config?.isZitiEnabled()) { + return; + } + if (!this.proxy) { + return; + } + await this.proxy.stop(); + } + + private async initialize(): Promise { + await this.reconciler.reconcile(); + await this.proxy.start(); + this.logger.log('Ziti control-plane reconciled'); + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.identity.manager.ts b/packages/platform-server/src/infra/ziti/ziti.identity.manager.ts new file mode 100644 index 000000000..f4c868085 --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.identity.manager.ts @@ -0,0 +1,70 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +import type { ZitiIdentityProfile } from './ziti.types'; +import type { ZitiManagementClient } from './ziti.management.client'; + +type ZitiSdk = typeof import('@openziti/ziti-sdk-nodejs'); + +@Injectable() +export class ZitiIdentityManager { + private readonly logger = new Logger(ZitiIdentityManager.name); + + async ensureIdentityMaterial(options: { + profile: ZitiIdentityProfile; + identityId: string; + enrollmentTtlSeconds: number; + directories: { identities: string; tmp: string }; + client: ZitiManagementClient; + }): Promise { + const { profile, identityId, enrollmentTtlSeconds, directories, client } = options; + const identityExists = await this.identityFileExists(profile.file); + if (identityExists) { + this.logger.log(`Ziti identity already exists for ${profile.name}`); + return; + } + + await this.ensureDirectories(directories, profile.file); + const expiresAt = new Date(Date.now() + enrollmentTtlSeconds * 1000).toISOString(); + const enrollment = await client.createEnrollment({ identityId, method: 'ott', expiresAt }); + if (!enrollment?.jwt) { + throw new Error(`Ziti enrollment for ${profile.name} did not include a JWT`); + } + + await this.writeIdentityFile(enrollment.jwt, profile.file, directories.tmp); + this.logger.log(`Generated Ziti identity for ${profile.name}`); + } + + private async identityFileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } + } + + private async ensureDirectories( + directories: { identities: string; tmp: string }, + destinationFile: string, + ): Promise { + await fs.mkdir(directories.identities, { recursive: true }); + await fs.mkdir(directories.tmp, { recursive: true }); + const destinationDir = path.dirname(destinationFile); + await fs.mkdir(destinationDir, { recursive: true }); + } + + private async writeIdentityFile(jwt: string, destination: string, tmpDir: string): Promise { + const tempFile = path.join(tmpDir, `${path.basename(destination)}.${Date.now()}.jwt`); + await fs.writeFile(tempFile, jwt, { encoding: 'utf8', mode: 0o600 }); + try { + const ziti = (await import('@openziti/ziti-sdk-nodejs')) as ZitiSdk; + const identity: unknown = await ziti.enroll(tempFile); + const serialized = JSON.stringify(identity, null, 2); + await fs.writeFile(destination, serialized, { encoding: 'utf8', mode: 0o640 }); + } finally { + await fs.rm(tempFile, { force: true }); + } + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.management.client.ts b/packages/platform-server/src/infra/ziti/ziti.management.client.ts new file mode 100644 index 000000000..9fb386eaf --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.management.client.ts @@ -0,0 +1,234 @@ +import { Logger } from '@nestjs/common'; +import { Agent, fetch } from 'undici'; + +import type { + ZitiEdgeRouter, + ZitiEdgeRouterPolicy, + ZitiEnrollment, + ZitiIdentity, + ZitiService, + ZitiServicePolicy, +} from './ziti.types'; + +type RequestOptions = { + body?: unknown; + searchParams?: Record; + auth?: boolean; +}; + +type Envelope = { + data: T; +}; + +type ListEnvelope = { + data: T[]; +}; + +type AuthenticationResponse = { + token: string; +}; + +export type ZitiManagementClientOptions = { + baseUrl: string; + username: string; + password: string; + insecureTls: boolean; +}; + +type ZitiServiceCreate = Partial & { + name: string; + encryptionRequired?: boolean; + terminatorStrategy?: string; +}; + +export class ZitiManagementClient { + private readonly logger = new Logger(ZitiManagementClient.name); + private readonly dispatcher: Agent; + private sessionToken?: string; + + constructor(private readonly options: ZitiManagementClientOptions) { + this.dispatcher = new Agent({ + connect: { + rejectUnauthorized: !options.insecureTls, + }, + }); + } + + async close(): Promise { + await this.dispatcher.close(); + } + + async authenticate(): Promise { + const response = await this.request>('POST', '/authenticate', { + auth: false, + searchParams: { method: 'password' }, + body: { + username: this.options.username, + password: this.options.password, + }, + }); + if (!response?.data?.token) { + throw new Error('Failed to authenticate against Ziti controller'); + } + this.sessionToken = response.data.token; + } + + async getServiceByName(name: string): Promise { + return this.findByName('/services', name); + } + + async createService(payload: ZitiServiceCreate): Promise { + const response = await this.request>('POST', '/services', { body: payload }); + return response.data; + } + + async updateService(id: string, payload: Partial): Promise { + const response = await this.request>('PATCH', `/services/${id}`, { body: payload }); + return response.data; + } + + async getServicePolicyByName(name: string): Promise { + return this.findByName('/service-policies', name); + } + + async createServicePolicy(payload: Omit): Promise { + const response = await this.request>('POST', '/service-policies', { body: payload }); + return response.data; + } + + async updateServicePolicy(id: string, payload: Partial): Promise { + const response = await this.request>('PATCH', `/service-policies/${id}`, { + body: payload, + }); + return response.data; + } + + async getServiceEdgeRouterPolicyByName(name: string): Promise { + return this.findByName('/service-edge-router-policies', name); + } + + async createServiceEdgeRouterPolicy( + payload: Omit, + ): Promise { + const response = await this.request>('POST', '/service-edge-router-policies', { + body: payload, + }); + return response.data; + } + + async updateServiceEdgeRouterPolicy( + id: string, + payload: Partial, + ): Promise { + const response = await this.request>('PATCH', `/service-edge-router-policies/${id}`, { + body: payload, + }); + return response.data; + } + + async getEdgeRouterByName(name: string): Promise { + return this.findByName('/edge-routers', name); + } + + async updateEdgeRouter(id: string, payload: Partial): Promise { + const response = await this.request>('PATCH', `/edge-routers/${id}`, { body: payload }); + return response.data; + } + + async getIdentityByName(name: string): Promise { + return this.findByName('/identities', name); + } + + async createIdentity(payload: Omit & { type: string; isAdmin: boolean }): Promise { + const response = await this.request>('POST', '/identities', { body: payload }); + return response.data; + } + + async updateIdentity(id: string, payload: Partial): Promise { + const response = await this.request>('PATCH', `/identities/${id}`, { body: payload }); + return response.data; + } + + async listIdentityEnrollments(identityId: string): Promise { + const response = await this.request>( + 'GET', + `/identities/${identityId}/enrollments`, + {}, + ); + return response.data; + } + + async createEnrollment(payload: { + identityId: string; + method: 'ott'; + expiresAt: string; + }): Promise { + const response = await this.request>('POST', '/enrollments', { body: payload }); + return response.data; + } + + private async findByName(path: string, name: string): Promise { + const filter = this.buildNameFilter(name); + const response = await this.request>('GET', path, { + searchParams: { filter }, + }); + return response.data?.[0]; + } + + private buildNameFilter(name: string): string { + const escaped = name.replace(/"/g, '\\"'); + return `name="${escaped}"`; + } + + private async request(method: string, path: string, options: RequestOptions = {}): Promise { + const url = new URL(path, this.options.baseUrl); + if (options.searchParams) { + for (const [key, value] of Object.entries(options.searchParams)) { + if (typeof value === 'string' && value.length > 0) { + url.searchParams.set(key, value); + } + } + } + + const headers: Record = {}; + let body: string | undefined; + if (options.body !== undefined) { + headers['content-type'] = 'application/json'; + body = JSON.stringify(options.body); + } + const shouldAuth = options.auth !== false; + if (shouldAuth) { + if (!this.sessionToken) { + throw new Error('Ziti session token missing; call authenticate() first'); + } + headers['zt-session'] = this.sessionToken; + } + + const response = await fetch(url, { + method, + body, + headers, + dispatcher: this.dispatcher, + }); + + if (!response.ok) { + const details = await this.safeReadError(response); + throw new Error(`Ziti management request failed (${response.status} ${response.statusText}): ${details}`); + } + + if (response.status === 204) { + return undefined as T; + } + return (await response.json()) as T; + } + + private async safeReadError(response: Response): Promise { + try { + const body = await response.text(); + return body || 'no body'; + } catch (error) { + this.logger.warn({ error }, 'failed to read Ziti error payload'); + return 'unavailable'; + } + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts new file mode 100644 index 000000000..1a6f300fe --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -0,0 +1,247 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { ConfigService } from '../../core/services/config.service'; +import { ZitiIdentityManager } from './ziti.identity.manager'; +import { ZitiManagementClient } from './ziti.management.client'; +import type { + ZitiEdgeRouter, + ZitiEdgeRouterPolicy, + ZitiIdentity, + ZitiIdentityProfile, + ZitiRuntimeProfile, + ZitiService, + ZitiServicePolicy, +} from './ziti.types'; + +const APP_ATTRIBUTE = 'app.agyn-platform'; +const SERVICE_ATTRIBUTE = 'service.platform-api'; +const ROUTER_ATTRIBUTE = 'router.platform'; +const PLATFORM_ATTRIBUTE = 'component.platform-server'; +const RUNNER_ATTRIBUTE = 'component.docker-runner'; + +@Injectable() +export class ZitiReconciler { + private readonly logger = new Logger(ZitiReconciler.name); + + constructor( + private readonly config: ConfigService, + private readonly identityManager: ZitiIdentityManager, + ) {} + + async reconcile(): Promise { + if (!this.config.isZitiEnabled()) { + this.logger.log('Ziti disabled; skipping controller reconciliation'); + return; + } + + const profile = this.buildProfile(); + const client = new ZitiManagementClient({ + baseUrl: profile.managementUrl, + username: profile.username, + password: profile.password, + insecureTls: profile.insecureTls, + }); + + try { + await client.authenticate(); + await this.ensureRouter(client, profile); + await this.ensureService(client, profile); + await this.ensureServicePolicies(client, profile); + await this.ensureEdgeRouterPolicy(client, profile); + + const platformIdentity = await this.ensureIdentity(client, profile.identities.platform); + const runnerIdentity = await this.ensureIdentity(client, profile.identities.runner); + + await this.identityManager.ensureIdentityMaterial({ + profile: profile.identities.platform, + identityId: platformIdentity.id, + enrollmentTtlSeconds: profile.enrollmentTtlSeconds, + directories: profile.directories, + client, + }); + + await this.identityManager.ensureIdentityMaterial({ + profile: profile.identities.runner, + identityId: runnerIdentity.id, + enrollmentTtlSeconds: profile.enrollmentTtlSeconds, + directories: profile.directories, + client, + }); + } finally { + await client.close(); + } + } + + private buildProfile(): ZitiRuntimeProfile { + const platformIdentity = this.config.getZitiPlatformIdentity(); + const runnerIdentity = this.config.getZitiRunnerIdentity(); + const directories = { + identities: this.config.getZitiIdentityDirectory(), + tmp: this.config.getZitiTmpDirectory(), + }; + const credentials = this.config.getZitiCredentials(); + return { + managementUrl: this.config.getZitiManagementUrl(), + username: credentials.username, + password: credentials.password, + insecureTls: this.config.getZitiInsecureTls(), + serviceName: this.config.getZitiServiceName(), + serviceRoleAttributes: [APP_ATTRIBUTE, SERVICE_ATTRIBUTE], + serviceSelectors: [`#${SERVICE_ATTRIBUTE}`], + routerName: this.config.getZitiRouterName(), + routerRoleAttributes: [ROUTER_ATTRIBUTE], + routerSelectors: [`#${ROUTER_ATTRIBUTE}`], + enrollmentTtlSeconds: this.config.getZitiEnrollmentTtlSeconds(), + directories, + runnerProxy: { + host: this.config.getZitiRunnerProxyHost(), + port: this.config.getZitiRunnerProxyPort(), + }, + identities: { + platform: { + name: platformIdentity.name, + file: platformIdentity.file, + roleAttributes: [APP_ATTRIBUTE, PLATFORM_ATTRIBUTE], + selectors: [`#${PLATFORM_ATTRIBUTE}`], + }, + runner: { + name: runnerIdentity.name, + file: runnerIdentity.file, + roleAttributes: [APP_ATTRIBUTE, RUNNER_ATTRIBUTE], + selectors: [`#${RUNNER_ATTRIBUTE}`], + }, + }, + }; + } + + private async ensureRouter(client: ZitiManagementClient, profile: ZitiRuntimeProfile): Promise { + const router = await client.getEdgeRouterByName(profile.routerName); + if (!router) { + throw new Error(`Ziti edge router "${profile.routerName}" was not found. Ensure docker-compose is running.`); + } + const missingAttrs = profile.routerRoleAttributes.filter((attr) => !router.roleAttributes?.includes(attr)); + if (missingAttrs.length === 0) { + return router; + } + const nextAttributes = Array.from(new Set([...(router.roleAttributes ?? []), ...missingAttrs])); + this.logger.log(`Updating Ziti router role attributes for ${profile.routerName}`); + return client.updateEdgeRouter(router.id, { roleAttributes: nextAttributes }); + } + + private async ensureService(client: ZitiManagementClient, profile: ZitiRuntimeProfile): Promise { + const existing = await client.getServiceByName(profile.serviceName); + if (!existing) { + this.logger.log(`Creating Ziti service ${profile.serviceName}`); + return client.createService({ + name: profile.serviceName, + encryptionRequired: true, + terminatorStrategy: 'smartrouting', + roleAttributes: profile.serviceRoleAttributes, + }); + } + const missingAttrs = profile.serviceRoleAttributes.filter((attr) => !existing.roleAttributes?.includes(attr)); + if (missingAttrs.length === 0) { + return existing; + } + this.logger.log(`Updating Ziti service attributes for ${profile.serviceName}`); + const nextAttributes = Array.from(new Set([...(existing.roleAttributes ?? []), ...missingAttrs])); + return client.updateService(existing.id, { roleAttributes: nextAttributes }); + } + + private async ensureServicePolicies(client: ZitiManagementClient, profile: ZitiRuntimeProfile): Promise { + await this.ensureServicePolicy(client, { + name: `${profile.serviceName}.dial`, + type: 'Dial', + semantic: 'AllOf', + identityRoles: profile.identities.runner.selectors, + serviceRoles: profile.serviceSelectors, + }); + await this.ensureServicePolicy(client, { + name: `${profile.serviceName}.bind`, + type: 'Bind', + semantic: 'AllOf', + identityRoles: profile.identities.platform.selectors, + serviceRoles: profile.serviceSelectors, + }); + } + + private async ensureServicePolicy( + client: ZitiManagementClient, + payload: Omit, + ): Promise { + const existing = await client.getServicePolicyByName(payload.name); + if (!existing) { + this.logger.log(`Creating Ziti service policy ${payload.name}`); + return client.createServicePolicy(payload); + } + + const needsUpdate = + existing.type !== payload.type || + existing.semantic !== payload.semantic || + !this.matches(existing.identityRoles, payload.identityRoles) || + !this.matches(existing.serviceRoles, payload.serviceRoles); + + if (!needsUpdate) { + return existing; + } + + this.logger.log(`Updating Ziti service policy ${payload.name}`); + return client.updateServicePolicy(existing.id, payload); + } + + private async ensureEdgeRouterPolicy( + client: ZitiManagementClient, + profile: ZitiRuntimeProfile, + ): Promise { + const payload: Omit = { + name: `${profile.serviceName}.edge-router`, + semantic: 'AllOf', + edgeRouterRoles: profile.routerSelectors, + serviceRoles: profile.serviceSelectors, + }; + const existing = await client.getServiceEdgeRouterPolicyByName(payload.name); + if (!existing) { + this.logger.log(`Creating Ziti edge-router policy ${payload.name}`); + return client.createServiceEdgeRouterPolicy(payload); + } + const needsUpdate = + existing.semantic !== payload.semantic || + !this.matches(existing.edgeRouterRoles, payload.edgeRouterRoles) || + !this.matches(existing.serviceRoles, payload.serviceRoles); + if (!needsUpdate) { + return existing; + } + this.logger.log(`Updating Ziti edge-router policy ${payload.name}`); + return client.updateServiceEdgeRouterPolicy(existing.id, payload); + } + + private async ensureIdentity( + client: ZitiManagementClient, + profile: ZitiIdentityProfile, + ): Promise { + const existing = await client.getIdentityByName(profile.name); + if (!existing) { + this.logger.log(`Creating Ziti identity ${profile.name}`); + return client.createIdentity({ + name: profile.name, + isAdmin: false, + type: 'Device', + roleAttributes: profile.roleAttributes, + }); + } + + const missingAttrs = profile.roleAttributes.filter((attr) => !existing.roleAttributes?.includes(attr)); + if (missingAttrs.length === 0) { + return existing; + } + this.logger.log(`Updating Ziti identity attributes for ${profile.name}`); + const nextAttributes = Array.from(new Set([...(existing.roleAttributes ?? []), ...missingAttrs])); + return client.updateIdentity(existing.id, { roleAttributes: nextAttributes }); + } + + private matches(current: string[] | undefined, desired: string[]): boolean { + if (!current) return false; + const currentSet = new Set(current); + return desired.every((value) => currentSet.has(value)); + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts new file mode 100644 index 000000000..53254ca47 --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts @@ -0,0 +1,123 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import type { Duplex } from 'node:stream'; +import httpProxy from 'http-proxy'; + +import { ConfigService } from '../../core/services/config.service'; + +type ZitiSdk = typeof import('@openziti/ziti-sdk-nodejs'); +type HttpProxyServer = ReturnType; + +@Injectable() +export class ZitiRunnerProxyService { + private readonly logger = new Logger(ZitiRunnerProxyService.name); + private server?: HttpServer; + private proxy?: HttpProxyServer; + private started = false; + + constructor(private readonly config: ConfigService) {} + + async start(): Promise { + if (!this.config.isZitiEnabled()) { + return; + } + if (this.started) { + return; + } + + const identity = this.config.getZitiPlatformIdentity(); + const ziti = (await import('@openziti/ziti-sdk-nodejs')) as ZitiSdk; + await ziti.init(identity.file); + const agent = ziti.httpAgent(); + + const proxy = httpProxy.createProxyServer({ + target: `http://${this.config.getZitiServiceName()}`, + agent, + changeOrigin: true, + ws: true, + }); + this.proxy = proxy; + + proxy.on('error', (error: Error, req?: IncomingMessage) => { + const context = { error: error.message, url: req?.url }; + this.logger.error(context, 'Ziti proxy error'); + }); + + const host = this.config.getZitiRunnerProxyHost(); + const port = this.config.getZitiRunnerProxyPort(); + + this.server = createServer((req, res) => { + if (!this.proxy) { + res.statusCode = 502; + res.end('Ziti proxy unavailable'); + return; + } + const activeProxy = this.proxy!; + activeProxy.web(req, res, undefined, (error: Error | null) => this.handleProxyFailure(error, res)); + }); + + this.server.on('upgrade', (req: IncomingMessage, socket: Duplex, head: Buffer) => { + if (!this.proxy) { + socket.destroy(); + return; + } + const activeProxy = this.proxy!; + activeProxy.ws(req, socket, head, undefined, (error: Error | null) => { + if (error) { + this.logger.error({ error: error.message, url: req.url }, 'Ziti proxy websocket failure'); + } + socket.destroy(); + }); + }); + + await new Promise((resolve, reject) => { + if (!this.server) { + reject(new Error('Ziti proxy server missing')); + return; + } + this.server.once('error', reject); + this.server.listen({ host, port }, () => { + this.server?.off('error', reject); + resolve(); + }); + }); + + this.started = true; + this.logger.log(`Ziti runner proxy listening on http://${host}:${port}`); + } + + async stop(): Promise { + if (!this.started) return; + await Promise.all([ + this.closeServer(), + this.closeProxy(), + ]); + this.started = false; + } + + private async closeServer(): Promise { + if (!this.server) return; + await new Promise((resolve) => { + this.server?.close(() => resolve()); + }); + this.server = undefined; + } + + private async closeProxy(): Promise { + if (!this.proxy) return; + const proxy = this.proxy!; + proxy.close(); + this.proxy = undefined; + } + + private handleProxyFailure(error: Error | null | undefined, res: ServerResponse): void { + if (res.headersSent) { + res.end(); + return; + } + this.logger.error({ error: error?.message }, 'Ziti proxy HTTP failure'); + res.statusCode = 502; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ error: 'ziti_proxy_error' })); + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.types.ts b/packages/platform-server/src/infra/ziti/ziti.types.ts new file mode 100644 index 000000000..ac39bd675 --- /dev/null +++ b/packages/platform-server/src/infra/ziti/ziti.types.ts @@ -0,0 +1,75 @@ +export type ZitiIdentityProfile = { + name: string; + file: string; + roleAttributes: string[]; + selectors: string[]; +}; + +export type ZitiRuntimeProfile = { + managementUrl: string; + username: string; + password: string; + insecureTls: boolean; + serviceName: string; + serviceRoleAttributes: string[]; + serviceSelectors: string[]; + routerName: string; + routerRoleAttributes: string[]; + routerSelectors: string[]; + enrollmentTtlSeconds: number; + directories: { + identities: string; + tmp: string; + }; + runnerProxy: { + host: string; + port: number; + }; + identities: { + platform: ZitiIdentityProfile; + runner: ZitiIdentityProfile; + }; +}; + +export type ZitiService = { + id: string; + name: string; + roleAttributes: string[]; +}; + +export type ZitiIdentity = { + id: string; + name: string; + roleAttributes: string[]; +}; + +export type ZitiEdgeRouter = { + id: string; + name: string; + roleAttributes: string[]; +}; + +export type ZitiServicePolicy = { + id: string; + name: string; + type: 'Bind' | 'Dial'; + semantic: 'AllOf' | 'AnyOf'; + identityRoles: string[]; + serviceRoles: string[]; +}; + +export type ZitiEdgeRouterPolicy = { + id: string; + name: string; + semantic: 'AllOf' | 'AnyOf'; + edgeRouterRoles: string[]; + serviceRoles: string[]; +}; + +export type ZitiEnrollment = { + id: string; + identityId: string; + method: string; + expiresAt: string; + jwt?: string; +}; diff --git a/packages/platform-server/src/types/openziti.d.ts b/packages/platform-server/src/types/openziti.d.ts new file mode 100644 index 000000000..7c3dc5432 --- /dev/null +++ b/packages/platform-server/src/types/openziti.d.ts @@ -0,0 +1,5 @@ +declare module '@openziti/ziti-sdk-nodejs' { + export function init(identityPath: string): Promise; + export function httpAgent(): unknown; + export function enroll(jwtPath: string): Promise; +} From e1a4c01024056e99e6859dff94bdb68c1aff9fd5 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Wed, 18 Feb 2026 03:01:06 +0000 Subject: [PATCH 02/22] fix(ziti): address review feedback --- README.md | 2 +- docs/containers/ziti.md | 2 +- .../docker-runner/src/service/ziti.ingress.ts | 11 ++++++- .../src/infra/ziti/ziti.management.client.ts | 24 ++++++++++++++ .../src/infra/ziti/ziti.reconciler.ts | 31 +++++++++++++++++++ .../src/infra/ziti/ziti.types.ts | 8 +++++ 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a021e946c..649e2662f 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ overlay instead of the Docker bridge network: 1. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). 2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). -3. Start the controller stack: `docker compose up -d ziti-controller controller-init ziti-edge-router`. +3. Start the controller stack: `docker compose up -d ziti-controller ziti-controller-init ziti-edge-router`. 4. Launch docker-runner and platform-server normally. The server will reconcile the controller, enroll identities, and expose a local proxy on `127.0.0.1:17071` for all docker-runner calls. diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index d0f92ec9f..cc4c47370 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -23,7 +23,7 @@ pnpm approve-builds 2. Ensure the dev stack is running: ```bash -docker compose up -d ziti-controller controller-init ziti-edge-router +docker compose up -d ziti-controller ziti-controller-init ziti-edge-router ``` 3. Copy `.env` files and enable OpenZiti flags: diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts index 3e3672e53..f3ef8a325 100644 --- a/packages/docker-runner/src/service/ziti.ingress.ts +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -19,7 +19,8 @@ export async function startZitiIngress(config: RunnerConfig): Promise => new Promise((resolve) => { server.close(() => resolve()); }); + +const resolveTargetHost = (host: string): string => { + const normalized = host.trim(); + if (!normalized || normalized === '0.0.0.0' || normalized === '::' || normalized === '[::]') { + return '127.0.0.1'; + } + return normalized; +}; diff --git a/packages/platform-server/src/infra/ziti/ziti.management.client.ts b/packages/platform-server/src/infra/ziti/ziti.management.client.ts index 9fb386eaf..67bb95817 100644 --- a/packages/platform-server/src/infra/ziti/ziti.management.client.ts +++ b/packages/platform-server/src/infra/ziti/ziti.management.client.ts @@ -6,6 +6,7 @@ import type { ZitiEdgeRouterPolicy, ZitiEnrollment, ZitiIdentity, + ZitiIdentityRouterPolicy, ZitiService, ZitiServicePolicy, } from './ziti.types'; @@ -126,6 +127,29 @@ export class ZitiManagementClient { return response.data; } + async getEdgeRouterPolicyByName(name: string): Promise { + return this.findByName('/edge-router-policies', name); + } + + async createEdgeRouterPolicy( + payload: Omit, + ): Promise { + const response = await this.request>('POST', '/edge-router-policies', { + body: payload, + }); + return response.data; + } + + async updateEdgeRouterPolicy( + id: string, + payload: Partial, + ): Promise { + const response = await this.request>('PATCH', `/edge-router-policies/${id}`, { + body: payload, + }); + return response.data; + } + async getEdgeRouterByName(name: string): Promise { return this.findByName('/edge-routers', name); } diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index 1a6f300fe..f1044f697 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -8,6 +8,7 @@ import type { ZitiEdgeRouterPolicy, ZitiIdentity, ZitiIdentityProfile, + ZitiIdentityRouterPolicy, ZitiRuntimeProfile, ZitiService, ZitiServicePolicy, @@ -48,6 +49,7 @@ export class ZitiReconciler { await this.ensureService(client, profile); await this.ensureServicePolicies(client, profile); await this.ensureEdgeRouterPolicy(client, profile); + await this.ensureIdentityRouterPolicy(client, profile); const platformIdentity = await this.ensureIdentity(client, profile.identities.platform); const runnerIdentity = await this.ensureIdentity(client, profile.identities.runner); @@ -215,6 +217,35 @@ export class ZitiReconciler { return client.updateServiceEdgeRouterPolicy(existing.id, payload); } + private async ensureIdentityRouterPolicy( + client: ZitiManagementClient, + profile: ZitiRuntimeProfile, + ): Promise { + const identityRoles = Array.from( + new Set([...profile.identities.platform.selectors, ...profile.identities.runner.selectors]), + ); + const payload: Omit = { + name: `${profile.serviceName}.identities.use-router`, + semantic: 'AnyOf', + identityRoles, + edgeRouterRoles: profile.routerSelectors, + }; + const existing = await client.getEdgeRouterPolicyByName(payload.name); + if (!existing) { + this.logger.log(`Creating Ziti identity-router policy ${payload.name}`); + return client.createEdgeRouterPolicy(payload); + } + const needsUpdate = + existing.semantic !== payload.semantic || + !this.matches(existing.identityRoles, payload.identityRoles) || + !this.matches(existing.edgeRouterRoles, payload.edgeRouterRoles); + if (!needsUpdate) { + return existing; + } + this.logger.log(`Updating Ziti identity-router policy ${payload.name}`); + return client.updateEdgeRouterPolicy(existing.id, payload); + } + private async ensureIdentity( client: ZitiManagementClient, profile: ZitiIdentityProfile, diff --git a/packages/platform-server/src/infra/ziti/ziti.types.ts b/packages/platform-server/src/infra/ziti/ziti.types.ts index ac39bd675..6ed69997a 100644 --- a/packages/platform-server/src/infra/ziti/ziti.types.ts +++ b/packages/platform-server/src/infra/ziti/ziti.types.ts @@ -66,6 +66,14 @@ export type ZitiEdgeRouterPolicy = { serviceRoles: string[]; }; +export type ZitiIdentityRouterPolicy = { + id: string; + name: string; + semantic: 'AllOf' | 'AnyOf'; + identityRoles: string[]; + edgeRouterRoles: string[]; +}; + export type ZitiEnrollment = { id: string; identityId: string; From 32be8e9992c8f523646bf68b6827778dfd0293c5 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Wed, 18 Feb 2026 03:08:51 +0000 Subject: [PATCH 03/22] fix(ziti): address typecheck regressions --- .../src/core/services/config.service.ts | 63 ++++++++++++------- .../src/infra/ziti/ziti.management.client.ts | 4 +- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/packages/platform-server/src/core/services/config.service.ts b/packages/platform-server/src/core/services/config.service.ts index 306504217..542b1c583 100644 --- a/packages/platform-server/src/core/services/config.service.ts +++ b/packages/platform-server/src/core/services/config.service.ts @@ -28,6 +28,25 @@ const numberFlag = (defaultValue: number) => const trimUrl = (value: string): string => value.trim().replace(/\/+$/, ''); +const defaultZitiConfig = { + enabled: false, + managementUrl: 'https://127.0.0.1:1280/edge/management/v1', + username: 'admin', + password: 'admin', + insecureTls: true, + serviceName: 'dev.agyn-platform.platform-api', + routerName: 'dev-edge-router', + runnerProxyHost: '127.0.0.1', + runnerProxyPort: 17071, + platformIdentityName: 'dev.agyn-platform.platform-server', + platformIdentityFile: '.ziti/identities/dev.agyn-platform.platform-server.json', + runnerIdentityName: 'dev.agyn-platform.docker-runner', + runnerIdentityFile: '.ziti/identities/dev.agyn-platform.docker-runner.json', + identitiesDir: '.ziti/identities', + tmpDir: '.ziti/tmp', + enrollmentTtlSeconds: 900, +} as const; + export const configSchema = z.object({ // GitHub settings are optional to allow dev boot without GitHub githubAppId: z.string().min(1).optional(), @@ -215,31 +234,27 @@ export const configSchema = z.object({ ), ziti: z .object({ - enabled: booleanFlag(false), + enabled: booleanFlag(defaultZitiConfig.enabled), managementUrl: z .string() - .default('https://127.0.0.1:1280/edge/management/v1') + .default(defaultZitiConfig.managementUrl) .transform((value) => trimUrl(value)), - username: z.string().default('admin'), - password: z.string().default('admin'), - insecureTls: booleanFlag(true), - serviceName: z.string().default('dev.agyn-platform.platform-api'), - routerName: z.string().default('dev-edge-router'), - runnerProxyHost: z.string().default('127.0.0.1'), - runnerProxyPort: numberFlag(17071), - platformIdentityName: z.string().default('dev.agyn-platform.platform-server'), - platformIdentityFile: z - .string() - .default('.ziti/identities/dev.agyn-platform.platform-server.json'), - runnerIdentityName: z.string().default('dev.agyn-platform.docker-runner'), - runnerIdentityFile: z - .string() - .default('.ziti/identities/dev.agyn-platform.docker-runner.json'), - identitiesDir: z.string().default('.ziti/identities'), - tmpDir: z.string().default('.ziti/tmp'), - enrollmentTtlSeconds: numberFlag(900), + username: z.string().default(defaultZitiConfig.username), + password: z.string().default(defaultZitiConfig.password), + insecureTls: booleanFlag(defaultZitiConfig.insecureTls), + serviceName: z.string().default(defaultZitiConfig.serviceName), + routerName: z.string().default(defaultZitiConfig.routerName), + runnerProxyHost: z.string().default(defaultZitiConfig.runnerProxyHost), + runnerProxyPort: numberFlag(defaultZitiConfig.runnerProxyPort), + platformIdentityName: z.string().default(defaultZitiConfig.platformIdentityName), + platformIdentityFile: z.string().default(defaultZitiConfig.platformIdentityFile), + runnerIdentityName: z.string().default(defaultZitiConfig.runnerIdentityName), + runnerIdentityFile: z.string().default(defaultZitiConfig.runnerIdentityFile), + identitiesDir: z.string().default(defaultZitiConfig.identitiesDir), + tmpDir: z.string().default(defaultZitiConfig.tmpDir), + enrollmentTtlSeconds: numberFlag(defaultZitiConfig.enrollmentTtlSeconds), }) - .default({}), + .default(() => ({ ...defaultZitiConfig })), }); export type Config = z.infer; @@ -474,10 +489,14 @@ export class ConfigService implements Config { return this.params.nixRepoAllowlist ?? []; } - get zitiConfig(): Config['ziti'] { + get ziti(): Config['ziti'] { return this.params.ziti; } + get zitiConfig(): Config['ziti'] { + return this.ziti; + } + isZitiEnabled(): boolean { return !!this.params.ziti?.enabled; } diff --git a/packages/platform-server/src/infra/ziti/ziti.management.client.ts b/packages/platform-server/src/infra/ziti/ziti.management.client.ts index 67bb95817..c04f244ee 100644 --- a/packages/platform-server/src/infra/ziti/ziti.management.client.ts +++ b/packages/platform-server/src/infra/ziti/ziti.management.client.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common'; -import { Agent, fetch } from 'undici'; +import { Agent, fetch, type Response as UndiciResponse } from 'undici'; import type { ZitiEdgeRouter, @@ -246,7 +246,7 @@ export class ZitiManagementClient { return (await response.json()) as T; } - private async safeReadError(response: Response): Promise { + private async safeReadError(response: UndiciResponse): Promise { try { const body = await response.text(); return body || 'no body'; From efdd928db5eae323ea589132da07963c47e66f75 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Wed, 18 Feb 2026 03:17:41 +0000 Subject: [PATCH 04/22] fix(docker-runner): default ziti config --- packages/docker-runner/src/service/config.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/docker-runner/src/service/config.ts b/packages/docker-runner/src/service/config.ts index 642871f35..e9eeca3d2 100644 --- a/packages/docker-runner/src/service/config.ts +++ b/packages/docker-runner/src/service/config.ts @@ -13,6 +13,12 @@ const booleanFlag = (defaultValue: boolean) => return defaultValue; }); +const defaultZitiConfig = { + enabled: false, + identityFile: '.ziti/identities/dev.agyn-platform.docker-runner.json', + serviceName: 'dev.agyn-platform.platform-api', +} as const; + const runnerConfigSchema = z.object({ port: z .union([z.string(), z.number()]) @@ -34,11 +40,11 @@ const runnerConfigSchema = z.object({ logLevel: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), ziti: z .object({ - enabled: booleanFlag(false), - identityFile: z.string().default('.ziti/identities/dev.agyn-platform.docker-runner.json'), - serviceName: z.string().default('dev.agyn-platform.platform-api'), + enabled: booleanFlag(defaultZitiConfig.enabled), + identityFile: z.string().default(defaultZitiConfig.identityFile), + serviceName: z.string().default(defaultZitiConfig.serviceName), }) - .default({}), + .default(() => ({ ...defaultZitiConfig })), }); export type RunnerConfig = z.infer; From 8a5524e5e6b3f720834cb8fe98de4dbfd039b686 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Wed, 18 Feb 2026 18:20:24 +0000 Subject: [PATCH 05/22] fix(ziti): stabilize runner proxy ingress --- .../docker-runner/src/service/ziti.ingress.ts | 67 ++++++++++++-- .../docker-runner/src/types/openziti.d.ts | 8 ++ .../src/infra/ziti/ziti.reconciler.ts | 45 ++++++++-- .../infra/ziti/ziti.runnerProxy.service.ts | 89 +++++++++++++++++-- 4 files changed, 191 insertions(+), 18 deletions(-) diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts index f3ef8a325..a8423928c 100644 --- a/packages/docker-runner/src/service/ziti.ingress.ts +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -4,12 +4,23 @@ import type { Duplex } from 'node:stream'; import express, { type Express } from 'express'; import httpProxy from 'http-proxy'; -import type { RunnerConfig } from './config'; +import type { RunnerConfig } from './config.js'; type ZitiIngressHandle = { close: () => Promise; }; +type ZitiExpressListenerModule = { + Server?: { + prototype: { + listen: (...args: unknown[]) => unknown; + }; + }; + default?: ZitiExpressListenerModule; +}; + +let zitiExpressPatched = false; + export async function startZitiIngress(config: RunnerConfig): Promise { if (!config.ziti.enabled) { return undefined; @@ -17,6 +28,7 @@ export async function startZitiIngress(config: RunnerConfig): Promise socket.destroy()); }); - // eslint-disable-next-line no-console -- CLI startup log console.info(`Ziti ingress ready for service ${config.ziti.serviceName}`); return { @@ -53,10 +64,24 @@ export async function startZitiIngress(config: RunnerConfig): Promise => new Promise((resolve, reject) => { const handleError = (error: unknown) => reject(error instanceof Error ? error : new Error('ziti ingress failed')); - const server = app.listen(() => { - server.off('error', handleError); - resolve(server); - }); + let server: HttpServer | undefined; + try { + server = app.listen(() => { + server?.off('error', handleError); + if (!server) { + reject(new Error('ziti express server unavailable')); + return; + } + resolve(server); + }) as HttpServer | undefined; + } catch (error) { + handleError(error); + return; + } + if (!server) { + handleError(new Error('ziti express listener did not return a server instance')); + return; + } server.once('error', handleError); }); @@ -72,3 +97,33 @@ const resolveTargetHost = (host: string): string => { } return normalized; }; + +async function ensureZitiExpressServerPatch(): Promise { + if (zitiExpressPatched) { + return; + } + const module = (await import('@openziti/ziti-sdk-nodejs/lib/express-listener.js')) as unknown as ZitiExpressListenerModule; + const listener = module.Server ?? module.default?.Server; + if (!listener) { + throw new Error('Failed to load OpenZiti express listener'); + } + const originalListen = listener.prototype.listen; + if (typeof originalListen !== 'function') { + throw new Error('OpenZiti express listener missing listen implementation'); + } + if ((originalListen as { __agynReturnsServer?: boolean }).__agynReturnsServer) { + zitiExpressPatched = true; + return; + } + listener.prototype.listen = function patchedListen(this: unknown, ...args: unknown[]) { + originalListen.apply(this, args as []); + return this; + }; + Object.defineProperty(listener.prototype.listen, '__agynReturnsServer', { + value: true, + configurable: false, + enumerable: false, + writable: false, + }); + zitiExpressPatched = true; +} diff --git a/packages/docker-runner/src/types/openziti.d.ts b/packages/docker-runner/src/types/openziti.d.ts index e99b1fe0a..1cf831c36 100644 --- a/packages/docker-runner/src/types/openziti.d.ts +++ b/packages/docker-runner/src/types/openziti.d.ts @@ -6,3 +6,11 @@ declare module '@openziti/ziti-sdk-nodejs' { export function enroll(jwtPath: string): Promise; export function express(appFactory: typeof import('express'), serviceName: string): Express; } + +declare module '@openziti/ziti-sdk-nodejs/lib/express-listener.js' { + export class Server { + prototype: { + listen: (...args: unknown[]) => unknown; + }; + } +} diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index f1044f697..dbe7fb46c 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -1,4 +1,6 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '../../core/services/config.service'; import { ZitiIdentityManager } from './ziti.identity.manager'; @@ -19,13 +21,17 @@ const SERVICE_ATTRIBUTE = 'service.platform-api'; const ROUTER_ATTRIBUTE = 'router.platform'; const PLATFORM_ATTRIBUTE = 'component.platform-server'; const RUNNER_ATTRIBUTE = 'component.docker-runner'; +const ROUTER_DISCOVERY_TIMEOUT_MS = 120_000; +const ROUTER_DISCOVERY_INTERVAL_MS = 2_000; @Injectable() export class ZitiReconciler { private readonly logger = new Logger(ZitiReconciler.name); constructor( + @Inject(ConfigService) private readonly config: ConfigService, + @Inject(ZitiIdentityManager) private readonly identityManager: ZitiIdentityManager, ) {} @@ -117,10 +123,7 @@ export class ZitiReconciler { } private async ensureRouter(client: ZitiManagementClient, profile: ZitiRuntimeProfile): Promise { - const router = await client.getEdgeRouterByName(profile.routerName); - if (!router) { - throw new Error(`Ziti edge router "${profile.routerName}" was not found. Ensure docker-compose is running.`); - } + const router = await this.waitForRouter(client, profile); const missingAttrs = profile.routerRoleAttributes.filter((attr) => !router.roleAttributes?.includes(attr)); if (missingAttrs.length === 0) { return router; @@ -130,6 +133,34 @@ export class ZitiReconciler { return client.updateEdgeRouter(router.id, { roleAttributes: nextAttributes }); } + private async waitForRouter( + client: ZitiManagementClient, + profile: ZitiRuntimeProfile, + ): Promise { + const deadline = Date.now() + ROUTER_DISCOVERY_TIMEOUT_MS; + let lastError: Error | undefined; + + while (Date.now() < deadline) { + try { + const router = await client.getEdgeRouterByName(profile.routerName); + if (router) { + return router; + } + lastError = new Error(`router ${profile.routerName} not yet registered`); + } catch (error) { + lastError = error as Error; + } + + this.logger.log( + `Waiting for Ziti router ${profile.routerName} to register (retrying in ${ROUTER_DISCOVERY_INTERVAL_MS / 1000}s)`, + ); + await delay(ROUTER_DISCOVERY_INTERVAL_MS); + } + + const reason = lastError?.message ?? 'unknown reason'; + throw new Error(`Ziti edge router "${profile.routerName}" was not found: ${reason}`); + } + private async ensureService(client: ZitiManagementClient, profile: ZitiRuntimeProfile): Promise { const existing = await client.getServiceByName(profile.serviceName); if (!existing) { @@ -155,14 +186,14 @@ export class ZitiReconciler { name: `${profile.serviceName}.dial`, type: 'Dial', semantic: 'AllOf', - identityRoles: profile.identities.runner.selectors, + identityRoles: profile.identities.platform.selectors, serviceRoles: profile.serviceSelectors, }); await this.ensureServicePolicy(client, { name: `${profile.serviceName}.bind`, type: 'Bind', semantic: 'AllOf', - identityRoles: profile.identities.platform.selectors, + identityRoles: profile.identities.runner.selectors, serviceRoles: profile.serviceSelectors, }); } diff --git a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts index 53254ca47..13a0fe4b8 100644 --- a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts +++ b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts @@ -1,6 +1,17 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { EventEmitter } from 'node:events'; +import { + createServer, + request as httpRequest, + type Agent as HttpAgent, + type IncomingMessage, + type Server as HttpServer, + type ServerResponse, +} from 'node:http'; +import { promises as fs, constants as fsConstants } from 'node:fs'; +import path from 'node:path'; import type { Duplex } from 'node:stream'; +import { setTimeout as delay } from 'node:timers/promises'; import httpProxy from 'http-proxy'; import { ConfigService } from '../../core/services/config.service'; @@ -15,7 +26,7 @@ export class ZitiRunnerProxyService { private proxy?: HttpProxyServer; private started = false; - constructor(private readonly config: ConfigService) {} + constructor(@Inject(ConfigService) private readonly config: ConfigService) {} async start(): Promise { if (!this.config.isZitiEnabled()) { @@ -26,9 +37,17 @@ export class ZitiRunnerProxyService { } const identity = this.config.getZitiPlatformIdentity(); + const identityFile = path.resolve(identity.file); + await this.ensureIdentityReadable(identityFile); + const tmpDir = path.resolve(this.config.getZitiTmpDirectory()); + await this.ensureWritableDirectory(tmpDir); const ziti = (await import('@openziti/ziti-sdk-nodejs')) as ZitiSdk; - await ziti.init(identity.file); - const agent = ziti.httpAgent(); + await ziti.init(identityFile); + const agent = ziti.httpAgent() as HttpAgent & EventEmitter; + agent.on('error', (error: Error) => { + this.logger.error({ error: error.message }, 'Ziti HTTP agent error'); + }); + await this.waitForService(agent); const proxy = httpProxy.createProxyServer({ target: `http://${this.config.getZitiServiceName()}`, @@ -70,6 +89,7 @@ export class ZitiRunnerProxyService { }); }); + this.logger.log(`Starting Ziti runner proxy on http://${host}:${port}`); await new Promise((resolve, reject) => { if (!this.server) { reject(new Error('Ziti proxy server missing')); @@ -120,4 +140,63 @@ export class ZitiRunnerProxyService { res.setHeader('content-type', 'application/json'); res.end(JSON.stringify({ error: 'ziti_proxy_error' })); } + + private async ensureIdentityReadable(file: string): Promise { + try { + await fs.access(file, fsConstants.R_OK); + } catch (error) { + this.logger.error({ file, error: (error as Error).message }, 'Ziti identity file missing or unreadable'); + throw new Error(`Ziti identity file missing or unreadable: ${file}`); + } + } + + private async ensureWritableDirectory(dir: string): Promise { + await fs.mkdir(dir, { recursive: true }); + await fs.access(dir, fsConstants.W_OK); + } + + private async waitForService(agent: HttpAgent): Promise { + const serviceHost = this.config.getZitiServiceName(); + const maxAttempts = 10; + const delayMs = 2000; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + await new Promise((resolve, reject) => { + const handleError = (error: Error) => { + reject(error); + }; + const req = httpRequest( + { + host: serviceHost, + agent, + method: 'HEAD', + timeout: 5000, + }, + (res) => { + res.resume(); + res.once('end', resolve); + res.once('error', handleError); + }, + ); + req.on('error', handleError); + req.on('timeout', () => { + req.destroy(new Error('timeout')); + }); + req.on('socket', (socket) => { + socket.on('error', handleError); + }); + req.end(); + }); + this.logger.log(`Ziti service reachable (${serviceHost})`); + return; + } catch (error) { + this.logger.warn( + { attempt, error: (error as Error).message }, + 'Ziti service not reachable yet; retrying', + ); + await delay(delayMs); + } + } + throw new Error(`Ziti service ${serviceHost} did not become reachable`); + } } From 72f7305fbcabcdc296b345c2ae77128b40fb3816 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 15:26:48 +0000 Subject: [PATCH 06/22] fix(platform-server): harden ziti runner proxy --- .../infra/ziti/ziti.runnerProxy.service.ts | 136 ++++++++++++++++-- 1 file changed, 126 insertions(+), 10 deletions(-) diff --git a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts index 13a0fe4b8..baf3b3256 100644 --- a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts +++ b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts @@ -4,7 +4,9 @@ import { createServer, request as httpRequest, type Agent as HttpAgent, + type IncomingHttpHeaders, type IncomingMessage, + type OutgoingHttpHeaders, type Server as HttpServer, type ServerResponse, } from 'node:http'; @@ -19,12 +21,28 @@ import { ConfigService } from '../../core/services/config.service'; type ZitiSdk = typeof import('@openziti/ziti-sdk-nodejs'); type HttpProxyServer = ReturnType; +const HOP_BY_HOP_HEADERS = new Set([ + 'connection', + 'proxy-connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]); + @Injectable() export class ZitiRunnerProxyService { private readonly logger = new Logger(ZitiRunnerProxyService.name); private server?: HttpServer; private proxy?: HttpProxyServer; + private agent?: (HttpAgent & EventEmitter) | null; + private serviceHost?: string; private started = false; + private requestSequence = 0; + private readonly traceEnabled = (process.env.ZITI_PROXY_TRACE ?? '').trim() === '1'; constructor(@Inject(ConfigService) private readonly config: ConfigService) {} @@ -47,10 +65,16 @@ export class ZitiRunnerProxyService { agent.on('error', (error: Error) => { this.logger.error({ error: error.message }, 'Ziti HTTP agent error'); }); + this.agent = agent; + this.serviceHost = this.config.getZitiServiceName(); await this.waitForService(agent); + if (this.traceEnabled) { + this.logger.log('Ziti runner proxy request tracing enabled'); + } + const proxy = httpProxy.createProxyServer({ - target: `http://${this.config.getZitiServiceName()}`, + target: `http://${this.serviceHost}`, agent, changeOrigin: true, ws: true, @@ -65,15 +89,7 @@ export class ZitiRunnerProxyService { const host = this.config.getZitiRunnerProxyHost(); const port = this.config.getZitiRunnerProxyPort(); - this.server = createServer((req, res) => { - if (!this.proxy) { - res.statusCode = 502; - res.end('Ziti proxy unavailable'); - return; - } - const activeProxy = this.proxy!; - activeProxy.web(req, res, undefined, (error: Error | null) => this.handleProxyFailure(error, res)); - }); + this.server = createServer((req, res) => this.handleHttpRequest(req, res)); this.server.on('upgrade', (req: IncomingMessage, socket: Duplex, head: Buffer) => { if (!this.proxy) { @@ -112,6 +128,8 @@ export class ZitiRunnerProxyService { this.closeServer(), this.closeProxy(), ]); + this.agent = null; + this.serviceHost = undefined; this.started = false; } @@ -155,6 +173,104 @@ export class ZitiRunnerProxyService { await fs.access(dir, fsConstants.W_OK); } + private handleHttpRequest(req: IncomingMessage, res: ServerResponse): void { + if (!this.agent || !this.serviceHost) { + this.handleProxyFailure(new Error('Ziti HTTP agent unavailable'), res); + return; + } + const requestId = this.nextRequestId(); + const method = req.method ?? 'GET'; + const url = req.url ?? '/'; + req.setTimeout(0); + res.setTimeout(0); + this.logTrace('proxy request', { id: requestId, method, url }); + + const headers = this.buildUpstreamRequestHeaders(req.headers); + this.logTrace('proxy forward', { + id: requestId, + method, + url, + 'content-length': req.headers['content-length'] ?? 'n/a', + 'upstream-content-length': headers['content-length'] ?? 'n/a', + }); + const upstreamReq = httpRequest( + { + host: this.serviceHost, + agent: this.agent, + method, + path: url, + headers, + }, + (upstreamRes) => { + this.logTrace('proxy response', { + id: requestId, + method, + url, + status: upstreamRes.statusCode ?? 0, + }); + const responseHeaders = this.buildDownstreamHeaders(upstreamRes.headers); + res.writeHead(upstreamRes.statusCode ?? 502, responseHeaders); + upstreamRes.pipe(res); + upstreamRes.once('close', () => { + this.logTrace('proxy upstream closed', { id: requestId, method, url }); + }); + }, + ); + + upstreamReq.on('error', (error) => { + this.logTrace('proxy error', { id: requestId, method, url, error: error.message }); + this.handleProxyFailure(error, res); + }); + + req.pipe(upstreamReq); + res.once('close', () => { + upstreamReq.destroy(new Error('downstream_closed')); + }); + req.once('aborted', () => { + upstreamReq.destroy(new Error('downstream_aborted')); + }); + } + + private buildUpstreamRequestHeaders(headers: IncomingHttpHeaders): OutgoingHttpHeaders { + const normalized: OutgoingHttpHeaders = {}; + for (const [key, value] of Object.entries(headers)) { + if (!key) continue; + if (HOP_BY_HOP_HEADERS.has(key.toLowerCase())) { + continue; + } + normalized[key] = value as string | string[]; + } + normalized.connection = 'close'; + if (this.serviceHost) { + normalized.host = this.serviceHost; + } + return normalized; + } + + private buildDownstreamHeaders(headers: IncomingHttpHeaders): OutgoingHttpHeaders { + const normalized: OutgoingHttpHeaders = {}; + for (const [key, value] of Object.entries(headers)) { + if (!key) continue; + if (HOP_BY_HOP_HEADERS.has(key.toLowerCase())) { + continue; + } + normalized[key] = value as string | string[]; + } + return normalized; + } + + private nextRequestId(): number { + this.requestSequence += 1; + return this.requestSequence; + } + + private logTrace(message: string, metadata: Record): void { + if (!this.traceEnabled) { + return; + } + this.logger.log(metadata, message); + } + private async waitForService(agent: HttpAgent): Promise { const serviceHost = this.config.getZitiServiceName(); const maxAttempts = 10; From c83e081d7d3b0f8261bba714b493cd03ccd39547 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 15:42:56 +0000 Subject: [PATCH 07/22] fix(ziti): restore platform bind policy --- packages/platform-server/src/infra/ziti/ziti.reconciler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index dbe7fb46c..952b78118 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -186,14 +186,14 @@ export class ZitiReconciler { name: `${profile.serviceName}.dial`, type: 'Dial', semantic: 'AllOf', - identityRoles: profile.identities.platform.selectors, + identityRoles: profile.identities.runner.selectors, serviceRoles: profile.serviceSelectors, }); await this.ensureServicePolicy(client, { name: `${profile.serviceName}.bind`, type: 'Bind', semantic: 'AllOf', - identityRoles: profile.identities.runner.selectors, + identityRoles: profile.identities.platform.selectors, serviceRoles: profile.serviceSelectors, }); } From 90de288dfbfc3f9a2bf008e49771e42a9aa6405c Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 22:43:31 +0000 Subject: [PATCH 08/22] chore(devops): split dev compose overlay --- README.md | 21 ++++++++--- docker-compose.dev.yml | 79 +++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 26 -------------- docs/containers/ziti.md | 28 ++++++++++----- 4 files changed, 114 insertions(+), 40 deletions(-) create mode 100644 docker-compose.dev.yml diff --git a/README.md b/README.md index 649e2662f..c5a382099 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ Intended use cases: - Operating a local development environment with supporting infra. ## Repository Structure -- docker-compose.yml — Development infra: Postgres, agents-db, Vault (+ auto-init), NCPS, LiteLLM, cAdvisor, Prometheus, Grafana. +- docker-compose.yml — Third-party development infra: Postgres, agents-db, Vault (+ auto-init), NCPS, LiteLLM, OpenZiti, Prometheus, Grafana, etc. +- docker-compose.dev.yml — Optional overlay that builds/runs the platform-server and docker-runner containers against the infra stack. - .github/workflows/ - ci.yml — Linting, tests (server/UI), Storybook build + smoke, type-check build steps. - docker-ghcr.yml — Build and publish platform-server and platform-ui images to GHCR. @@ -117,9 +118,14 @@ pnpm install ```bash docker compose up -d # Starts postgres (5442), agents-db (5443), vault (8200), ncps (8501), -# litellm (127.0.0.1:4000), docker-runner (7071), and the OpenZiti controller stack +# litellm (127.0.0.1:4000), registry-mirror, and the OpenZiti controller stack # Optional monitoring (prometheus/grafana) lives in docker-compose.monitoring.yml. # Enable with: docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d + +# Optional: run platform-server and docker-runner inside Docker as well. +mkdir -p .ziti/identities .ziti/tmp data/graph +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner +# The overlay builds local images and reuses the infra services defined in docker-compose.yml. ``` 4) Apply server migrations and generate Prisma client: @@ -141,6 +147,9 @@ pnpm --filter @agyn/platform-ui dev # docker-runner (Fastify dev server) pnpm --filter @agyn/docker-runner dev ``` +> Prefer containers? Use `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` +> after the infra stack is running. The overlay file builds the local images against the +> same Postgres/LiteLLM/Vault/Ziti services. Server listens on PORT (default 3010; see packages/platform-server/src/index.ts and Dockerfile), UI dev server on default Vite port. The docker-runner dev script automatically loads the first `.env` it finds (prefers repo root, falls back to packages/docker-runner) when `NODE_ENV` is not `production`. Production `pnpm start` keeps relying solely on the surrounding environment, so missing `.env` files do not crash the process. @@ -174,8 +183,10 @@ overlay instead of the Docker bridge network: 2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). 3. Start the controller stack: `docker compose up -d ziti-controller ziti-controller-init ziti-edge-router`. -4. Launch docker-runner and platform-server normally. The server will reconcile the controller, enroll identities, and - expose a local proxy on `127.0.0.1:17071` for all docker-runner calls. +4. Launch docker-runner and platform-server normally (either via `pnpm dev` or + `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). + The server will reconcile the controller, enroll identities, and expose a local proxy on `0.0.0.0:17071` + (typically accessed via `127.0.0.1`) for all docker-runner calls. See [docs/containers/ziti.md](docs/containers/ziti.md) for the full walkthrough and smoke test commands. @@ -239,8 +250,8 @@ UI variables (packages/platform-ui/.env.example): - vault — HashiCorp Vault (8200), auto-init helper vault-auto-init - ncps — Nix cache proxy (8501) - litellm + litellm-db — LLM proxy with UI (4000 loopback) - - docker-runner — authenticated Docker API proxy (7071, mounts /var/run/docker.sock) - Optional monitoring overlay (docker-compose.monitoring.yml) adds prometheus (9090) and grafana (3000) without mounting the Docker socket; provide your own scrape targets via configuration. +- In-repo services (platform-server, docker-runner) live in docker-compose.dev.yml and must be combined with the base file. To start services: ```bash diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000..98df91635 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,79 @@ +services: + platform-server: + build: + context: . + dockerfile: packages/platform-server/Dockerfile + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + agents-db: + condition: service_healthy + litellm: + condition: service_started + docker-runner: + condition: service_started + environment: + NODE_ENV: production + PORT: ${PLATFORM_SERVER_PORT:-3010} + AGENTS_DATABASE_URL: ${AGENTS_DATABASE_URL:-postgresql://agents:agents@agents-db:5432/agents} + LLM_PROVIDER: ${LLM_PROVIDER:-litellm} + LITELLM_BASE_URL: ${LITELLM_BASE_URL:-http://litellm:4000} + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-dev-master-1234} + DOCKER_RUNNER_BASE_URL: http://docker-runner:7071 + DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} + WORKSPACE_NETWORK_NAME: ${WORKSPACE_NETWORK_NAME:-agents_net} + VAULT_ADDR: ${VAULT_ADDR:-http://vault:8200} + VAULT_TOKEN: ${VAULT_TOKEN:-dev-root} + ZITI_ENABLED: ${ZITI_ENABLED:-false} + ZITI_MANAGEMENT_URL: ${ZITI_MANAGEMENT_URL:-https://ziti-controller:1280/edge/management/v1} + ZITI_USERNAME: ${ZITI_USERNAME:-admin} + ZITI_PASSWORD: ${ZITI_PASSWORD:-admin} + ZITI_INSECURE_TLS: ${ZITI_INSECURE_TLS:-true} + ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} + ZITI_ROUTER_NAME: ${ZITI_ROUTER_NAME:-dev-edge-router} + ZITI_RUNNER_PROXY_HOST: ${ZITI_RUNNER_PROXY_HOST:-0.0.0.0} + ZITI_RUNNER_PROXY_PORT: ${ZITI_RUNNER_PROXY_PORT:-17071} + ZITI_PLATFORM_IDENTITY_FILE: ${ZITI_PLATFORM_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.platform-server.json} + ZITI_RUNNER_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} + ZITI_IDENTITIES_DIR: ${ZITI_IDENTITIES_DIR:-/opt/app/.ziti/identities} + ZITI_TMP_DIR: ${ZITI_TMP_DIR:-/opt/app/.ziti/tmp} + ports: + - "${PLATFORM_SERVER_PORT:-3010}:3010" + - "${ZITI_RUNNER_PROXY_PORT:-17071}:17071" + volumes: + - type: bind + source: ./.ziti + target: /opt/app/.ziti + - type: bind + source: ./data/graph + target: /opt/app/data/graph + networks: + - agents_net + + docker-runner: + build: + context: . + dockerfile: packages/docker-runner/Dockerfile + restart: unless-stopped + depends_on: + ziti-edge-router: + condition: service_started + environment: + DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} + DOCKER_RUNNER_PORT: ${DOCKER_RUNNER_PORT:-7071} + ZITI_ENABLED: ${ZITI_ENABLED:-false} + ZITI_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} + ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: ./.ziti + target: /opt/app/.ziti + read_only: true + ports: + - "${DOCKER_RUNNER_PORT:-7071}:7071" + networks: + - agents_net diff --git a/docker-compose.yml b/docker-compose.yml index 85f83390c..7b696cf88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -334,32 +334,6 @@ services: networks: - agents_net - docker-runner: - build: - context: . - dockerfile: packages/docker-runner/Dockerfile - restart: unless-stopped - depends_on: - ziti-edge-router: - condition: service_started - environment: - DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} - DOCKER_RUNNER_PORT: ${DOCKER_RUNNER_PORT:-7071} - ZITI_ENABLED: ${ZITI_ENABLED:-false} - ZITI_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} - ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - - type: bind - source: ./.ziti/identities - target: /opt/app/.ziti/identities - read_only: true - ports: - - "${DOCKER_RUNNER_PORT:-7071}:7071" - networks: - - agents_net volumes: vault-file: diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index cc4c47370..4689d86cb 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -2,8 +2,9 @@ The local development stack now provisions an OpenZiti controller, initializer, and edge router. The platform-server reconciles controller state at startup (service, policies, and identities) and stores identity material under -`./.ziti/identities`. A lightweight local HTTP proxy (`127.0.0.1:17071`) tunnels docker-runner traffic through the -OpenZiti overlay instead of the Docker bridge network when enabled. +`./.ziti/identities` (mirrored to `/opt/app/.ziti/identities` inside containers). A lightweight local HTTP proxy binds to +`0.0.0.0:17071` (reachable via `127.0.0.1` from the same host) and tunnels docker-runner traffic through the OpenZiti +overlay instead of the Docker bridge network when enabled. ## Prerequisites @@ -20,12 +21,16 @@ pnpm approve-builds > pnpm --dir node_modules/.pnpm/@openziti+ziti-sdk-nodejs@0.27.0/node_modules/@openziti/ziti-sdk-nodejs run install > ``` -2. Ensure the dev stack is running: +2. Ensure the OpenZiti controller stack is running: ```bash docker compose up -d ziti-controller ziti-controller-init ziti-edge-router ``` +> Running platform-server and docker-runner inside Docker? After the infra stack is up, +> start them with `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` +> so they share the same controller and network. + 3. Copy `.env` files and enable OpenZiti flags: - `packages/platform-server/.env` @@ -38,8 +43,12 @@ ZITI_PASSWORD=admin ZITI_INSECURE_TLS=true ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ZITI_ROUTER_NAME=dev-edge-router -ZITI_PLATFORM_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.platform-server.json -ZITI_RUNNER_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_RUNNER_PROXY_HOST=0.0.0.0 +ZITI_RUNNER_PROXY_PORT=17071 +ZITI_PLATFORM_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITIES_DIR=/opt/app/.ziti/identities +ZITI_TMP_DIR=/tmp/ziti ``` - `packages/docker-runner/.env` (or container env) @@ -50,8 +59,9 @@ ZITI_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ``` -The docker-compose service already mounts `./.ziti` into both platform-server and docker-runner containers. Local -development outside Docker can re-use the same paths. +The docker-compose.dev.yml overlay mounts `./.ziti` into both platform-server and docker-runner containers (presented as +`/opt/app/.ziti/*` in each container). Local development outside Docker can re-use the same paths or override the env +vars with machine-local locations. ## Runtime flow @@ -60,7 +70,7 @@ development outside Docker can re-use the same paths. - Ensures bind/dial service policies and a service-edge-router policy targeting `dev-edge-router`. - Creates device identities for the server (`component.platform-server`) and docker-runner (`component.docker-runner`). - Generates OTT enrollments and writes identities to `.ziti/identities/`. -2. Ziti runner proxy starts on `127.0.0.1:17071` and dials the service using the platform-server identity. All requests to +2. Ziti runner proxy starts on `0.0.0.0:17071` (reachable via `127.0.0.1`) and dials the service using the platform-server identity. All requests to docker-runner are routed through this proxy when `ZITI_ENABLED=true`. 3. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an OpenZiti Express listener that proxies traffic to the local Fastify server. @@ -89,5 +99,5 @@ docker logs docker-runner | grep "Ziti ingress ready" Seeing the readiness log after step 1 indicates the end-to-end tunnel is operational. -> To reset the environment delete `./.ziti/identities` and `./.ziti/tmp`, then restart the stack so the platform-server +> To reset the environment delete `./.ziti/identities` and `./.ziti/tmp` (or the `/opt/app/.ziti/*` mounts inside Docker), then restart the stack so the platform-server > can re-enroll identities. From acc74445102d50b8b31862797567a356c09918d8 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 22:58:06 +0000 Subject: [PATCH 09/22] docs(ziti): clarify proxy bindings --- README.md | 7 +++-- docs/containers/ziti.md | 56 +++++++++++++++++++++++++++----------- docs/technical-overview.md | 5 +++- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c5a382099..14ecb81b9 100644 --- a/README.md +++ b/README.md @@ -185,8 +185,11 @@ overlay instead of the Docker bridge network: 3. Start the controller stack: `docker compose up -d ziti-controller ziti-controller-init ziti-edge-router`. 4. Launch docker-runner and platform-server normally (either via `pnpm dev` or `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). - The server will reconcile the controller, enroll identities, and expose a local proxy on `0.0.0.0:17071` - (typically accessed via `127.0.0.1`) for all docker-runner calls. + - Host `pnpm` dev keeps the ConfigService defaults, so the proxy binds to `127.0.0.1:17071` unless you override + `ZITI_RUNNER_PROXY_HOST`. + - The docker overlay sets `ZITI_RUNNER_PROXY_HOST=0.0.0.0` inside the container so port `17071` can be published to + the host. + In both cases traffic is accessible from the host via `127.0.0.1:17071` once the proxy reports ready. See [docs/containers/ziti.md](docs/containers/ziti.md) for the full walkthrough and smoke test commands. diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 4689d86cb..066391669 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -2,9 +2,10 @@ The local development stack now provisions an OpenZiti controller, initializer, and edge router. The platform-server reconciles controller state at startup (service, policies, and identities) and stores identity material under -`./.ziti/identities` (mirrored to `/opt/app/.ziti/identities` inside containers). A lightweight local HTTP proxy binds to -`0.0.0.0:17071` (reachable via `127.0.0.1` from the same host) and tunnels docker-runner traffic through the OpenZiti -overlay instead of the Docker bridge network when enabled. +`./.ziti/identities` (mirrored to `/opt/app/.ziti/identities` inside containers). When OpenZiti is enabled the +platform-server launches a lightweight HTTP proxy: `pnpm` dev binds to `127.0.0.1:17071` by default, while the +docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published to the host. +All docker-runner traffic is tunneled through this proxy instead of the Docker bridge network. ## Prerequisites @@ -31,7 +32,9 @@ docker compose up -d ziti-controller ziti-controller-init ziti-edge-router > start them with `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` > so they share the same controller and network. -3. Copy `.env` files and enable OpenZiti flags: +3. Copy `.env` files and enable OpenZiti flags. Use the template that matches how you run the services: + +### Host (`pnpm dev`) - `packages/platform-server/.env` @@ -43,25 +46,45 @@ ZITI_PASSWORD=admin ZITI_INSECURE_TLS=true ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ZITI_ROUTER_NAME=dev-edge-router -ZITI_RUNNER_PROXY_HOST=0.0.0.0 +ZITI_RUNNER_PROXY_HOST=127.0.0.1 ZITI_RUNNER_PROXY_PORT=17071 -ZITI_PLATFORM_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.platform-server.json -ZITI_RUNNER_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json -ZITI_IDENTITIES_DIR=/opt/app/.ziti/identities -ZITI_TMP_DIR=/tmp/ziti +ZITI_PLATFORM_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITIES_DIR=$(pwd)/.ziti/identities +ZITI_TMP_DIR=$(pwd)/.ziti/tmp ``` -- `packages/docker-runner/.env` (or container env) +- `packages/docker-runner/.env` ``` ZITI_ENABLED=true -ZITI_IDENTITY_FILE=./.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.docker-runner.json ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ``` -The docker-compose.dev.yml overlay mounts `./.ziti` into both platform-server and docker-runner containers (presented as -`/opt/app/.ziti/*` in each container). Local development outside Docker can re-use the same paths or override the env -vars with machine-local locations. +### Docker compose overlay (`docker-compose.dev.yml`) + +Compose already mounts `./.ziti` into `/opt/app/.ziti` inside each container. Override the same variables with +container paths (via `.env` or `docker-compose.dev.yml`): + +``` +ZITI_ENABLED=true +ZITI_MANAGEMENT_URL=https://ziti-controller:1280/edge/management/v1 +ZITI_USERNAME=admin +ZITI_PASSWORD=admin +ZITI_INSECURE_TLS=true +ZITI_SERVICE_NAME=dev.agyn-platform.platform-api +ZITI_ROUTER_NAME=dev-edge-router +ZITI_RUNNER_PROXY_HOST=0.0.0.0 +ZITI_RUNNER_PROXY_PORT=17071 +ZITI_PLATFORM_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITIES_DIR=/opt/app/.ziti/identities +ZITI_TMP_DIR=/opt/app/.ziti/tmp + +# docker-runner container +ZITI_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json +``` ## Runtime flow @@ -70,8 +93,9 @@ vars with machine-local locations. - Ensures bind/dial service policies and a service-edge-router policy targeting `dev-edge-router`. - Creates device identities for the server (`component.platform-server`) and docker-runner (`component.docker-runner`). - Generates OTT enrollments and writes identities to `.ziti/identities/`. -2. Ziti runner proxy starts on `0.0.0.0:17071` (reachable via `127.0.0.1`) and dials the service using the platform-server identity. All requests to - docker-runner are routed through this proxy when `ZITI_ENABLED=true`. +2. Ziti runner proxy starts on `127.0.0.1:17071` by default when running via `pnpm dev`. The docker-compose overlay + overrides it to `0.0.0.0:17071` inside the container so the port can be published. All requests to docker-runner are + routed through this proxy when `ZITI_ENABLED=true`. 3. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an OpenZiti Express listener that proxies traffic to the local Fastify server. diff --git a/docs/technical-overview.md b/docs/technical-overview.md index a9222f55f..7534e1331 100644 --- a/docs/technical-overview.md +++ b/docs/technical-overview.md @@ -70,7 +70,10 @@ Remote Docker runner - The platform-server always routes container lifecycle, exec, and log streaming calls through the `@agyn/docker-runner` service. - The runner exposes authenticated Fastify HTTP/SSE/WebSocket endpoints with HMAC headers derived solely from `DOCKER_RUNNER_SHARED_SECRET`. - Only the docker-runner service mounts `/var/run/docker.sock` in default stacks; platform-server and auxiliary services talk to it over the internal network (default http://docker-runner:7071). -- When `ZITI_ENABLED=true`, platform-server launches a local proxy on `127.0.0.1:17071` backed by the OpenZiti Node SDK and docker-runner binds the same API to the OpenZiti service (`dev.agyn-platform.platform-api`). +- When `ZITI_ENABLED=true`, platform-server launches a local proxy bound to `127.0.0.1:17071` by default (`pnpm dev`). + The docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published. The + proxy is backed by the OpenZiti Node SDK and docker-runner binds the same API to the OpenZiti service + (`dev.agyn-platform.platform-api`). - Container events are forwarded via SSE so the existing watcher pipeline (ContainerEventProcessor, cleanup jobs, metrics) remains unchanged. Defaults and toggles From fc64711e1c1a3d90e58d5d15b1b463d8338991a9 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 23:03:00 +0000 Subject: [PATCH 10/22] docs(ziti): provide literal env paths --- docs/containers/ziti.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 066391669..9333b85df 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -48,20 +48,24 @@ ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ZITI_ROUTER_NAME=dev-edge-router ZITI_RUNNER_PROXY_HOST=127.0.0.1 ZITI_RUNNER_PROXY_PORT=17071 -ZITI_PLATFORM_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.platform-server.json -ZITI_RUNNER_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.docker-runner.json -ZITI_IDENTITIES_DIR=$(pwd)/.ziti/identities -ZITI_TMP_DIR=$(pwd)/.ziti/tmp +ZITI_PLATFORM_IDENTITY_FILE=/absolute/path/to/platform/.ziti/identities/dev.agyn-platform.platform-server.json +ZITI_RUNNER_IDENTITY_FILE=/absolute/path/to/platform/.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITIES_DIR=/absolute/path/to/platform/.ziti/identities +ZITI_TMP_DIR=/absolute/path/to/platform/.ziti/tmp ``` - `packages/docker-runner/.env` ``` ZITI_ENABLED=true -ZITI_IDENTITY_FILE=$(pwd)/.ziti/identities/dev.agyn-platform.docker-runner.json +ZITI_IDENTITY_FILE=/absolute/path/to/platform/.ziti/identities/dev.agyn-platform.docker-runner.json ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ``` +> Replace `/absolute/path/to/platform` with your local repository root. When using shell exports instead of `.env` +> files you can set the variables with `$(pwd)` (e.g., +> `export ZITI_PLATFORM_IDENTITY_FILE="$(pwd)/.ziti/identities/..."`). + ### Docker compose overlay (`docker-compose.dev.yml`) Compose already mounts `./.ziti` into `/opt/app/.ziti` inside each container. Override the same variables with From be876d516b013077c4818214af2df3d917602c44 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Thu, 19 Feb 2026 23:19:11 +0000 Subject: [PATCH 11/22] chore(deps): refresh pnpm lock --- pnpm-lock.yaml | 449 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 449 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a66bbe527..d791ee97c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -69,15 +69,24 @@ importers: '@nestjs/common': specifier: ^11.1.7 version: 11.1.7(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@openziti/ziti-sdk-nodejs': + specifier: ^0.27.0 + version: 0.27.0 dockerode: specifier: ^4.0.8 version: 4.0.8 dotenv: specifier: ^17.2.2 version: 17.2.2 + express: + specifier: ^4.21.1 + version: 4.22.1 fastify: specifier: ^5.6.1 version: 5.6.1 + http-proxy: + specifier: ^1.18.1 + version: 1.18.1 node-fetch-native: specifier: ^1.6.7 version: 1.6.7 @@ -97,6 +106,12 @@ importers: '@types/dockerode': specifier: ^3.3.44 version: 3.3.44 + '@types/express': + specifier: ^4.17.21 + version: 4.17.25 + '@types/http-proxy': + specifier: ^1.17.14 + version: 1.17.17 '@types/node': specifier: ^24.5.1 version: 24.5.2 @@ -190,6 +205,9 @@ importers: '@octokit/rest': specifier: ^22.0.0 version: 22.0.0 + '@openziti/ziti-sdk-nodejs': + specifier: ^0.27.0 + version: 0.27.0 '@prisma/client': specifier: ^6.18.0 version: 6.18.0(prisma@6.18.0(typescript@5.8.3))(typescript@5.8.3) @@ -217,6 +235,9 @@ importers: fastify: specifier: ^5.6.1 version: 5.6.1 + http-proxy: + specifier: ^1.18.1 + version: 1.18.1 iconv-lite: specifier: ^0.6.3 version: 0.6.3 @@ -287,6 +308,9 @@ importers: '@nestjs/testing': specifier: ^11.1.8 version: 11.1.8(@nestjs/common@11.1.7(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.7(@nestjs/common@11.1.7(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)) + '@types/http-proxy': + specifier: ^1.17.14 + version: 1.17.17 '@types/lodash-es': specifier: ^4.17.12 version: 4.17.12 @@ -1812,6 +1836,11 @@ packages: resolution: {integrity: sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==} engines: {node: '>=8'} + '@mapbox/node-pre-gyp@2.0.3': + resolution: {integrity: sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg==} + engines: {node: '>=18'} + hasBin: true + '@mdx-js/react@3.1.1': resolution: {integrity: sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==} peerDependencies: @@ -2085,6 +2114,10 @@ packages: resolution: {integrity: sha512-JD6DerIKdJGmRp4jQyX5FlrQjA4tjOw1cvfsPAZXfOOEErMUHjPcPSICS+6WnM0nB0efSFARh0KAZss+bvExOA==} engines: {node: '>=14'} + '@openziti/ziti-sdk-nodejs@0.27.0': + resolution: {integrity: sha512-V02rML+QmtR+RksRFMYNcW7acqlLEP7KzzdOJ+pLrz0r47o4sMcMvrk0kSi7lbteT8zSj8zS66VojOZZah7Q1A==} + engines: {node: '>=20.0.0'} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -3686,9 +3719,15 @@ packages: '@types/babel__traverse@7.28.0': resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + '@types/body-parser@1.19.6': + resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} + '@types/chai@5.2.2': resolution: {integrity: sha512-8kB30R7Hwqf40JPiKhVzodJs2Qc1ZJ5zuT3uzw5Hq/dhNCl3G3l83jfpdI1e20BP348+fV7VIL/+FxaXkqBmWg==} + '@types/connect@3.4.38': + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} @@ -3809,6 +3848,12 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/express-serve-static-core@4.19.8': + resolution: {integrity: sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==} + + '@types/express@4.17.25': + resolution: {integrity: sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==} + '@types/geojson@7946.0.16': resolution: {integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==} @@ -3818,6 +3863,12 @@ packages: '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/http-errors@2.0.5': + resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==} + + '@types/http-proxy@1.17.17': + resolution: {integrity: sha512-ED6LB+Z1AVylNTu7hdzuBqOgMnvG/ld6wGCG8wFnAzKX5uyW2K3WD52v0gnLCTK/VLpXtKckgWuyScYK6cSPaw==} + '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} @@ -3848,6 +3899,9 @@ packages: '@types/mdx@2.0.13': resolution: {integrity: sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==} + '@types/mime@1.3.5': + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} @@ -3863,6 +3917,12 @@ packages: '@types/node@24.5.2': resolution: {integrity: sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ==} + '@types/qs@6.14.0': + resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==} + + '@types/range-parser@1.2.7': + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + '@types/react-dom@19.1.9': resolution: {integrity: sha512-qXRuZaOsAdXKFyOhRBg6Lqqc0yay13vN7KrIg4L7N4aaHN68ma9OK3NE1BoDFgFOTfM7zg+3/8+2n8rLUH3OKQ==} peerDependencies: @@ -3883,6 +3943,15 @@ packages: '@types/semver@7.7.1': resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/send@0.17.6': + resolution: {integrity: sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==} + + '@types/send@1.2.1': + resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} + + '@types/serve-static@1.15.10': + resolution: {integrity: sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==} + '@types/socket.io@3.0.2': resolution: {integrity: sha512-pu0sN9m5VjCxBZVK8hW37ZcMe8rjn4HHggBN5CbaRTvFwv5jOmuIRZEuddsBPa9Th0ts0SIo3Niukq+95cMBbQ==} deprecated: This is a stub types definition. socket.io provides its own type definitions, so you do not need this installed. @@ -4166,6 +4235,10 @@ packages: '@xyflow/system@0.0.73': resolution: {integrity: sha512-C2ymH2V4mYDkdVSiRx0D7R0s3dvfXiupVBcko6tXP5K4tVdSBMo22/e3V9yRNdn+2HQFv44RFKzwOyCcUUDAVQ==} + abbrev@3.0.1: + resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} + engines: {node: ^18.17.0 || >=20.5.0} + abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -4279,6 +4352,9 @@ packages: resolution: {integrity: sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==} engines: {node: '>= 0.4'} + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + asn1@0.2.6: resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} @@ -4383,9 +4459,16 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + body-parser@1.20.4: + resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + body-parser@2.2.0: resolution: {integrity: sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==} engines: {node: '>=18'} @@ -4663,6 +4746,10 @@ packages: console-table-printer@2.14.6: resolution: {integrity: sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw==} + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + content-disposition@1.0.0: resolution: {integrity: sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==} engines: {node: '>= 0.6'} @@ -4677,6 +4764,9 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie-signature@1.0.7: + resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} + cookie-signature@1.2.2: resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} engines: {node: '>=6.6.0'} @@ -4807,6 +4897,14 @@ packages: date-fns@3.6.0: resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} engines: {node: '>=6.0'} @@ -4907,6 +5005,10 @@ packages: destr@2.0.5: resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==} + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.1.0: resolution: {integrity: sha512-vEtk+OcP7VBRtQZ1EJ3bdgzSfBjgnEalLTp5zjJrS+2Z1w2KZly4SBdac/WDU3hhsNAZ9E8SC96ME4Ey8MZ7cg==} engines: {node: '>=8'} @@ -5253,6 +5355,10 @@ packages: peerDependencies: express: '>= 4.11' + express@4.22.1: + resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} + engines: {node: '>= 0.10.0'} + express@5.1.0: resolution: {integrity: sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==} engines: {node: '>= 18'} @@ -5359,6 +5465,9 @@ packages: resolution: {integrity: sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg==} engines: {node: '>=20'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + filesize@10.1.6: resolution: {integrity: sha512-sJslQKU2uM33qH5nqewAwVB2QgR6w1aMNsYUp3aN5rMRyXEwJGmZvaWzeJFNTOXWlHQyBFCWrdj3fV/fsTOX8w==} engines: {node: '>= 10.4.0'} @@ -5367,6 +5476,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + finalhandler@1.3.2: + resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} + engines: {node: '>= 0.8'} + finalhandler@2.1.0: resolution: {integrity: sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==} engines: {node: '>= 0.8'} @@ -5445,6 +5558,10 @@ packages: fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + fresh@2.0.0: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} @@ -5672,10 +5789,18 @@ packages: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + http-proxy@1.18.1: + resolution: {integrity: sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==} + engines: {node: '>=8.0.0'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -5688,6 +5813,10 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -6490,10 +6619,17 @@ packages: mdn-data@2.12.2: resolution: {integrity: sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + media-typer@1.1.0: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + merge-descriptors@2.0.0: resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} engines: {node: '>=18'} @@ -6505,6 +6641,10 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} @@ -6609,6 +6749,11 @@ packages: resolution: {integrity: sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==} engines: {node: '>= 0.6'} + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -6651,6 +6796,9 @@ packages: monaco-editor@0.53.0: resolution: {integrity: sha512-0WNThgC6CMWNXXBxTbaYYcunj08iB5rnx4/G56UOPeL9UVIUGGHA1GR0EWIh9Ebabj7NpCRawQ5b0hfN1jQmYQ==} + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -6723,6 +6871,15 @@ packages: node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -6733,6 +6890,11 @@ packages: node-releases@2.0.21: resolution: {integrity: sha512-5b0pgg78U3hwXkCM8Z9b2FJdPZlr9Psr9V2gQPESdGHqbntyFJKFW4r5TeWGFzafGY3hzs1JC62VEQMbl1JFkw==} + nopt@8.1.0: + resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} + engines: {node: ^18.17.0 || >=20.5.0} + hasBin: true + normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -6929,6 +7091,9 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + path-to-regexp@6.3.0: resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} @@ -7203,6 +7368,10 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + raw-body@3.0.1: resolution: {integrity: sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==} engines: {node: '>= 0.10'} @@ -7439,6 +7608,9 @@ packages: require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} + requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -7541,10 +7713,18 @@ packages: engines: {node: '>=10'} hasBin: true + send@0.19.2: + resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} + engines: {node: '>= 0.8.0'} + send@1.2.0: resolution: {integrity: sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==} engines: {node: '>= 18'} + serve-static@1.16.3: + resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} + engines: {node: '>= 0.8.0'} + serve-static@2.2.0: resolution: {integrity: sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==} engines: {node: '>= 18'} @@ -7937,6 +8117,9 @@ packages: resolution: {integrity: sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==} engines: {node: '>=16'} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tr46@6.0.0: resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==} engines: {node: '>=20'} @@ -8021,6 +8204,10 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + type-is@2.0.1: resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} engines: {node: '>= 0.6'} @@ -8149,6 +8336,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} hasBin: true @@ -8291,6 +8482,9 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webidl-conversions@8.0.0: resolution: {integrity: sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA==} engines: {node: '>=20'} @@ -8311,6 +8505,9 @@ packages: resolution: {integrity: sha512-2ytDk0kiEj/yu90JOAp44PVPUkO9+jVhyf+SybKlRHSDlvOOZhdPIrr7xTH64l4WixO2cP+wQIcgujkGBPPz6g==} engines: {node: '>=20'} + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which-boxed-primitive@1.1.1: resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==} engines: {node: '>= 0.4'} @@ -9843,6 +10040,19 @@ snapshots: '@lukeed/csprng@1.1.0': {} + '@mapbox/node-pre-gyp@2.0.3': + dependencies: + consola: 3.4.2 + detect-libc: 2.1.0 + https-proxy-agent: 7.0.6 + node-fetch: 2.7.0 + nopt: 8.1.0 + semver: 7.7.2 + tar: 7.4.3 + transitivePeerDependencies: + - encoding + - supports-color + '@mdx-js/react@3.1.1(@types/react@19.1.13)(react@19.1.1)': dependencies: '@types/mdx': 2.0.13 @@ -10174,6 +10384,14 @@ snapshots: '@opentelemetry/semantic-conventions@1.37.0': optional: true + '@openziti/ziti-sdk-nodejs@0.27.0': + dependencies: + '@mapbox/node-pre-gyp': 2.0.3 + bindings: 1.5.0 + transitivePeerDependencies: + - encoding + - supports-color + '@pinojs/redact@0.4.0': {} '@pkgjs/parseargs@0.11.0': @@ -11786,10 +12004,19 @@ snapshots: dependencies: '@babel/types': 7.28.4 + '@types/body-parser@1.19.6': + dependencies: + '@types/connect': 3.4.38 + '@types/node': 24.5.2 + '@types/chai@5.2.2': dependencies: '@types/deep-eql': 4.0.2 + '@types/connect@3.4.38': + dependencies: + '@types/node': 24.5.2 + '@types/cookie@0.6.0': {} '@types/cors@2.8.19': @@ -11938,6 +12165,20 @@ snapshots: '@types/estree@1.0.8': {} + '@types/express-serve-static-core@4.19.8': + dependencies: + '@types/node': 24.5.2 + '@types/qs': 6.14.0 + '@types/range-parser': 1.2.7 + '@types/send': 1.2.1 + + '@types/express@4.17.25': + dependencies: + '@types/body-parser': 1.19.6 + '@types/express-serve-static-core': 4.19.8 + '@types/qs': 6.14.0 + '@types/serve-static': 1.15.10 + '@types/geojson@7946.0.16': {} '@types/hast@2.3.10': @@ -11948,6 +12189,12 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/http-errors@2.0.5': {} + + '@types/http-proxy@1.17.17': + dependencies: + '@types/node': 24.5.2 + '@types/istanbul-lib-coverage@2.0.6': {} '@types/istanbul-lib-report@3.0.3': @@ -11979,6 +12226,8 @@ snapshots: '@types/mdx@2.0.13': {} + '@types/mime@1.3.5': {} + '@types/ms@2.1.0': {} '@types/mustache@4.2.6': {} @@ -11995,6 +12244,10 @@ snapshots: dependencies: undici-types: 7.12.0 + '@types/qs@6.14.0': {} + + '@types/range-parser@1.2.7': {} + '@types/react-dom@19.1.9(@types/react@19.1.13)': dependencies: '@types/react': 19.1.13 @@ -12013,6 +12266,21 @@ snapshots: '@types/semver@7.7.1': {} + '@types/send@0.17.6': + dependencies: + '@types/mime': 1.3.5 + '@types/node': 24.5.2 + + '@types/send@1.2.1': + dependencies: + '@types/node': 24.5.2 + + '@types/serve-static@1.15.10': + dependencies: + '@types/http-errors': 2.0.5 + '@types/node': 24.5.2 + '@types/send': 0.17.6 + '@types/socket.io@3.0.2': dependencies: socket.io: 4.8.1 @@ -12349,6 +12617,8 @@ snapshots: d3-selection: 3.0.0 d3-zoom: 3.0.0 + abbrev@3.0.1: {} + abstract-logging@2.0.1: {} accepts@1.3.8: @@ -12456,6 +12726,8 @@ snapshots: call-bound: 1.0.4 is-array-buffer: 3.0.5 + array-flatten@1.1.1: {} + asn1@0.2.6: dependencies: safer-buffer: 2.1.2 @@ -12577,12 +12849,33 @@ snapshots: dependencies: require-from-string: 2.0.2 + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 + body-parser@1.20.4: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.14.0 + raw-body: 2.5.3 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + body-parser@2.2.0: dependencies: bytes: 3.1.2 @@ -12853,6 +13146,10 @@ snapshots: dependencies: simple-wcswidth: 1.1.2 + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + content-disposition@1.0.0: dependencies: safe-buffer: 5.2.1 @@ -12863,6 +13160,8 @@ snapshots: convert-source-map@2.0.0: {} + cookie-signature@1.0.7: {} + cookie-signature@1.2.2: {} cookie@0.7.2: {} @@ -12989,6 +13288,10 @@ snapshots: date-fns@3.6.0: {} + debug@2.6.9: + dependencies: + ms: 2.0.0 + debug@4.3.7: dependencies: ms: 2.1.3 @@ -13073,6 +13376,8 @@ snapshots: destr@2.0.5: {} + destroy@1.2.0: {} + detect-libc@2.1.0: {} detect-newline@3.1.0: {} @@ -13488,6 +13793,42 @@ snapshots: dependencies: express: 5.1.0 + express@4.22.1: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.4 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.0.7 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.2 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.12 + proxy-addr: 2.0.7 + qs: 6.14.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.2 + serve-static: 1.16.3 + setprototypeof: 1.2.0 + statuses: 2.0.2 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + express@5.1.0: dependencies: accepts: 2.0.0 @@ -13640,12 +13981,26 @@ snapshots: transitivePeerDependencies: - supports-color + file-uri-to-path@1.0.0: {} + filesize@10.1.6: {} fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 + finalhandler@1.3.2: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + finalhandler@2.1.0: dependencies: debug: 4.4.3 @@ -13733,6 +14088,8 @@ snapshots: fraction.js@4.3.7: {} + fresh@0.5.2: {} + fresh@2.0.0: {} fromentries@1.3.2: {} @@ -14014,6 +14371,14 @@ snapshots: statuses: 2.0.1 toidentifier: 1.0.1 + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -14021,6 +14386,14 @@ snapshots: transitivePeerDependencies: - supports-color + http-proxy@1.18.1: + dependencies: + eventemitter3: 4.0.7 + follow-redirects: 1.15.11 + requires-port: 1.0.0 + transitivePeerDependencies: + - debug + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -14032,6 +14405,10 @@ snapshots: human-signals@8.0.1: {} + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -15105,14 +15482,20 @@ snapshots: mdn-data@2.12.2: {} + media-typer@0.3.0: {} + media-typer@1.1.0: {} + merge-descriptors@1.0.3: {} + merge-descriptors@2.0.0: {} merge-stream@2.0.0: {} merge2@1.4.1: {} + methods@1.1.2: {} + micromark-core-commonmark@2.0.3: dependencies: decode-named-character-reference: 1.2.0 @@ -15321,6 +15704,8 @@ snapshots: dependencies: mime-db: 1.54.0 + mime@1.6.0: {} + mimic-fn@2.1.0: {} min-indent@1.0.1: {} @@ -15351,6 +15736,8 @@ snapshots: dependencies: '@types/trusted-types': 1.0.6 + ms@2.0.0: {} + ms@2.1.3: {} msw-storybook-addon@2.0.6(msw@2.11.3(@types/node@24.5.2)(typescript@5.8.3)): @@ -15448,6 +15835,10 @@ snapshots: node-fetch-native@1.6.7: {} + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-int64@0.4.0: {} node-preload@0.2.1: @@ -15456,6 +15847,10 @@ snapshots: node-releases@2.0.21: {} + nopt@8.1.0: + dependencies: + abbrev: 3.0.1 + normalize-path@3.0.0: {} normalize-range@0.1.2: {} @@ -15678,6 +16073,8 @@ snapshots: lru-cache: 10.4.3 minipass: 7.1.2 + path-to-regexp@0.1.12: {} + path-to-regexp@6.3.0: {} path-to-regexp@8.3.0: {} @@ -15962,6 +16359,13 @@ snapshots: range-parser@1.2.1: {} + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + raw-body@3.0.1: dependencies: bytes: 3.1.2 @@ -16268,6 +16672,8 @@ snapshots: require-main-filename@2.0.0: {} + requires-port@1.0.0: {} + resolve-cwd@3.0.0: dependencies: resolve-from: 5.0.0 @@ -16379,6 +16785,24 @@ snapshots: semver@7.7.2: {} + send@0.19.2: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.1 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + send@1.2.0: dependencies: debug: 4.4.3 @@ -16395,6 +16819,15 @@ snapshots: transitivePeerDependencies: - supports-color + serve-static@1.16.3: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.2 + transitivePeerDependencies: + - supports-color + serve-static@2.2.0: dependencies: encodeurl: 2.0.0 @@ -16839,6 +17272,8 @@ snapshots: dependencies: tldts: 7.0.16 + tr46@0.0.3: {} + tr46@6.0.0: dependencies: punycode: 2.3.1 @@ -16910,6 +17345,11 @@ snapshots: type-fest@4.41.0: {} + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + type-is@2.0.1: dependencies: content-type: 1.0.5 @@ -17057,6 +17497,8 @@ snapshots: util-deprecate@1.0.2: {} + utils-merge@1.0.1: {} + uuid@10.0.0: {} uuid@13.0.0: {} @@ -17306,6 +17748,8 @@ snapshots: web-namespaces@2.0.1: {} + webidl-conversions@3.0.1: {} + webidl-conversions@8.0.0: {} webpack-virtual-modules@0.6.2: {} @@ -17321,6 +17765,11 @@ snapshots: tr46: 6.0.0 webidl-conversions: 8.0.0 + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which-boxed-primitive@1.1.1: dependencies: is-bigint: 1.1.0 From 0beaa37d5c4683f37754f5ef669d2be4f71f6b99 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 00:38:47 +0000 Subject: [PATCH 12/22] feat(ziti): automate controller provisioning --- README.md | 13 +- docker-compose.yml | 29 ++- docs/containers/ziti.md | 44 ++-- .../src/infra/ziti/ziti.reconciler.ts | 4 +- scripts/ziti/controller-init.sh | 220 ++++++++++++++++++ 5 files changed, 283 insertions(+), 27 deletions(-) create mode 100755 scripts/ziti/controller-init.sh diff --git a/README.md b/README.md index 14ecb81b9..587398e4a 100644 --- a/README.md +++ b/README.md @@ -182,8 +182,17 @@ overlay instead of the Docker bridge network: 1. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). 2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). -3. Start the controller stack: `docker compose up -d ziti-controller ziti-controller-init ziti-edge-router`. -4. Launch docker-runner and platform-server normally (either via `pnpm dev` or +3. Start the controller stack: `docker compose up -d ziti-controller ziti-edge-router`. +4. Bootstrap the controller state (service, policies, identities) via the bundled init job: + +```bash +docker compose run --rm ziti-controller-init +``` + +The init container wraps the OpenZiti CLI, mirrors identity JSON into `./.ziti/identities`, and can be re-run +whenever you need to regenerate enrollment material (no host `ziti` binary required). + +5. Launch docker-runner and platform-server normally (either via `pnpm dev` or `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). - Host `pnpm` dev keeps the ConfigService defaults, so the proxy binds to `127.0.0.1:17071` unless you override `ZITI_RUNNER_PROXY_HOST`. diff --git a/docker-compose.yml b/docker-compose.yml index 7b696cf88..af7e96fdc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -291,16 +291,35 @@ services: ziti-controller: condition: service_healthy entrypoint: - - "/var/openziti/scripts/run-with-ziti-cli.sh" - command: - - "/var/openziti/scripts/access-control.sh" + - "/bin/bash" + - "/scripts/ziti/controller-init.sh" environment: ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} + ZITI_USER: ${ZITI_USER:-admin} + ZITI_PWD: ${ZITI_PWD:-admin} + ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} + ZITI_PLATFORM_IDENTITY_NAME: ${ZITI_PLATFORM_IDENTITY_NAME:-dev.agyn-platform.platform-server} + ZITI_RUNNER_IDENTITY_NAME: ${ZITI_RUNNER_IDENTITY_NAME:-dev.agyn-platform.docker-runner} + ZITI_ROUTER_NAME: ${ZITI_ROUTER_NAME:-dev-edge-router} + ZITI_ENROLLMENT_DURATION_MINUTES: ${ZITI_ENROLLMENT_DURATION_MINUTES:-1440} + ZITI_IDENTITIES_DIR: /identities + ZITI_IDENTITIES_TMP: /ziti-tmp + ZITI_PLATFORM_IDENTITY_FILE: ${ZITI_PLATFORM_IDENTITY_FILE:-/identities/dev.agyn-platform.platform-server.json} + ZITI_RUNNER_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/identities/dev.agyn-platform.docker-runner.json} volumes: - type: bind source: ./.ziti/controller target: /persistent + - type: bind + source: ./.ziti/identities + target: /identities + - type: bind + source: ./.ziti/tmp + target: /ziti-tmp + - type: bind + source: ./scripts/ziti + target: /scripts/ziti networks: - agents_net @@ -309,8 +328,8 @@ services: container_name: ziti-edge-router restart: unless-stopped depends_on: - ziti-controller-init: - condition: service_completed_successfully + ziti-controller: + condition: service_healthy entrypoint: /bin/bash command: "/var/openziti/scripts/run-router.sh edge" environment: diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 9333b85df..d7972da6c 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -1,11 +1,15 @@ # OpenZiti integration -The local development stack now provisions an OpenZiti controller, initializer, and edge router. The platform-server -reconciles controller state at startup (service, policies, and identities) and stores identity material under -`./.ziti/identities` (mirrored to `/opt/app/.ziti/identities` inside containers). When OpenZiti is enabled the -platform-server launches a lightweight HTTP proxy: `pnpm` dev binds to `127.0.0.1:17071` by default, while the -docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published to the host. -All docker-runner traffic is tunneled through this proxy instead of the Docker bridge network. +The local development stack now provisions an OpenZiti controller, initializer, and edge router. The +`ziti-controller-init` job wraps the OpenZiti CLI inside the official image so you never have to install `ziti` on the +host. It creates/updates the service, policies, and identities, then writes enrollment files to `./.ziti/identities` +(mirrored to `/opt/app/.ziti/identities` inside containers). Platform-server still reconciles controller state at +startup to heal drift, and the docker-runner retains the same service bindings. + +When OpenZiti is enabled the platform-server launches a lightweight HTTP proxy: `pnpm` dev binds to +`127.0.0.1:17071` by default, while the docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so +the port can be published to the host. All docker-runner traffic is tunneled through this proxy instead of the Docker +bridge network. ## Prerequisites @@ -25,14 +29,20 @@ pnpm approve-builds 2. Ensure the OpenZiti controller stack is running: ```bash -docker compose up -d ziti-controller ziti-controller-init ziti-edge-router +docker compose up -d ziti-controller ziti-edge-router ``` > Running platform-server and docker-runner inside Docker? After the infra stack is up, > start them with `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` > so they share the same controller and network. -3. Copy `.env` files and enable OpenZiti flags. Use the template that matches how you run the services: +3. Bootstrap the controller via the init job (idempotent; re-run whenever identity JSON needs to be regenerated): + +```bash +docker compose run --rm ziti-controller-init +``` + +4. Copy `.env` files and enable OpenZiti flags. Use the template that matches how you run the services: ### Host (`pnpm dev`) @@ -62,9 +72,7 @@ ZITI_IDENTITY_FILE=/absolute/path/to/platform/.ziti/identities/dev.agyn-platform ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ``` -> Replace `/absolute/path/to/platform` with your local repository root. When using shell exports instead of `.env` -> files you can set the variables with `$(pwd)` (e.g., -> `export ZITI_PLATFORM_IDENTITY_FILE="$(pwd)/.ziti/identities/..."`). +> Replace `/absolute/path/to/platform` with your local repository root (for example `/Users/casey/dev/platform`). ### Docker compose overlay (`docker-compose.dev.yml`) @@ -92,15 +100,15 @@ ZITI_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.jso ## Runtime flow -1. Platform-server bootstraps the controller via the Ziti Management API: - - Creates/updates the service (`dev.agyn-platform.platform-api`). - - Ensures bind/dial service policies and a service-edge-router policy targeting `dev-edge-router`. - - Creates device identities for the server (`component.platform-server`) and docker-runner (`component.docker-runner`). - - Generates OTT enrollments and writes identities to `.ziti/identities/`. -2. Ziti runner proxy starts on `127.0.0.1:17071` by default when running via `pnpm dev`. The docker-compose overlay +1. `docker compose run --rm ziti-controller-init` wraps the OpenZiti CLI to create/update the service, + policies, router bindings, and the two device identities, then writes the enrolled JSON files to `.ziti/identities/`. + The same directory is mounted into `/opt/app/.ziti/identities` when running inside Docker. +2. Platform-server still authenticates to the management API at startup to reconcile drift (service attributes, + policies, router roles) before launching the local proxy. If the identity files already exist they are reused as-is. +3. Ziti runner proxy starts on `127.0.0.1:17071` by default when running via `pnpm dev`. The docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published. All requests to docker-runner are routed through this proxy when `ZITI_ENABLED=true`. -3. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an +4. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an OpenZiti Express listener that proxies traffic to the local Fastify server. ## Smoke test diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index 952b78118..dbe7fb46c 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -186,14 +186,14 @@ export class ZitiReconciler { name: `${profile.serviceName}.dial`, type: 'Dial', semantic: 'AllOf', - identityRoles: profile.identities.runner.selectors, + identityRoles: profile.identities.platform.selectors, serviceRoles: profile.serviceSelectors, }); await this.ensureServicePolicy(client, { name: `${profile.serviceName}.bind`, type: 'Bind', semantic: 'AllOf', - identityRoles: profile.identities.platform.selectors, + identityRoles: profile.identities.runner.selectors, serviceRoles: profile.serviceSelectors, }); } diff --git a/scripts/ziti/controller-init.sh b/scripts/ziti/controller-init.sh new file mode 100755 index 000000000..e190dcede --- /dev/null +++ b/scripts/ziti/controller-init.sh @@ -0,0 +1,220 @@ +#!/bin/bash +set -euo pipefail + +log() { + printf '[ziti-controller-init] %s\n' "$1" +} + +ZITI_SCRIPTS=${ZITI_SCRIPTS:-/var/openziti/scripts} +ZITI_HOME=${ZITI_HOME:-/persistent} +IDENTITIES_DIR=${ZITI_IDENTITIES_DIR:-/identities} +TMP_DIR=${ZITI_IDENTITIES_TMP:-/ziti-tmp} +ROUTER_DISCOVERY_TIMEOUT=${ZITI_ROUTER_DISCOVERY_TIMEOUT_SECONDS:-120} +ROUTER_DISCOVERY_INTERVAL=${ZITI_ROUTER_DISCOVERY_INTERVAL_SECONDS:-2} +ENROLLMENT_DURATION_MINUTES=${ZITI_ENROLLMENT_DURATION_MINUTES:-1440} + +APP_ATTRIBUTE='app.agyn-platform' +SERVICE_ATTRIBUTE='service.platform-api' +ROUTER_ATTRIBUTE='router.platform' +PLATFORM_ATTRIBUTE='component.platform-server' +RUNNER_ATTRIBUTE='component.docker-runner' + +SERVICE_ROLE_ATTRIBUTES="${APP_ATTRIBUTE},${SERVICE_ATTRIBUTE}" +PLATFORM_ROLE_ATTRIBUTES="${APP_ATTRIBUTE},${PLATFORM_ATTRIBUTE}" +RUNNER_ROLE_ATTRIBUTES="${APP_ATTRIBUTE},${RUNNER_ATTRIBUTE}" +ROUTER_ROLE_ATTRIBUTES="${ROUTER_ATTRIBUTE}" + +SERVICE_ROLES="#${SERVICE_ATTRIBUTE}" +PLATFORM_ROLES="#${PLATFORM_ATTRIBUTE}" +RUNNER_ROLES="#${RUNNER_ATTRIBUTE}" +ROUTER_ROLES="#${ROUTER_ATTRIBUTE}" + +ZITI_SERVICE_NAME=${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} +ZITI_PLATFORM_IDENTITY_NAME=${ZITI_PLATFORM_IDENTITY_NAME:-dev.agyn-platform.platform-server} +ZITI_RUNNER_IDENTITY_NAME=${ZITI_RUNNER_IDENTITY_NAME:-dev.agyn-platform.docker-runner} +ZITI_ROUTER_NAME=${ZITI_ROUTER_NAME:-dev-edge-router} +ZITI_PLATFORM_IDENTITY_FILE=${ZITI_PLATFORM_IDENTITY_FILE:-${IDENTITIES_DIR}/${ZITI_PLATFORM_IDENTITY_NAME}.json} +ZITI_RUNNER_IDENTITY_FILE=${ZITI_RUNNER_IDENTITY_FILE:-${IDENTITIES_DIR}/${ZITI_RUNNER_IDENTITY_NAME}.json} + +umask 027 +mkdir -p "${IDENTITIES_DIR}" "${TMP_DIR}" + +wait_for_env_file() { + while [[ ! -f "${ZITI_HOME}/ziti.env" ]]; do + log "waiting for ${ZITI_HOME}/ziti.env" + sleep 1 + done + # give the controller a moment to finish writing the file + sleep 1 +} + +source_environment() { + wait_for_env_file + # shellcheck disable=SC1091 + source "${ZITI_SCRIPTS}/ziti-cli-functions.sh" + : "${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:=ziti-controller}" + : "${ZITI_CTRL_EDGE_ADVERTISED_PORT:=1280}" + : "${ZITI_USER:=admin}" + : "${ZITI_PWD:=admin}" + # shellcheck disable=SC1091 + source "${ZITI_HOME}/ziti.env" + _wait_for_controller + zitiLogin >/dev/null + ZITI_BIN=${ZITI_BIN_DIR:-/var/openziti/ziti-bin}/ziti +} + +entity_exists() { + local resource=$1 + local name=$2 + local filter + filter=$(printf 'name = "%s"' "$name") + local output + if ! output=$("${ZITI_BIN}" edge list "$resource" "$filter" -j 2>/dev/null); then + return 1 + fi + printf '%s' "$output" | grep -F "\"name\":\"${name}\"" >/dev/null +} + +ensure_access_control_seed() { + local marker="${ZITI_HOME}/access-control.init" + if [[ -f "$marker" ]]; then + log 'access-control defaults already applied' + return + fi + log 'applying OpenZiti access-control defaults' + "${ZITI_SCRIPTS}/access-control.sh" + touch "$marker" +} + +wait_for_router() { + local deadline=$(( $(date +%s) + ROUTER_DISCOVERY_TIMEOUT )) + while (( $(date +%s) < deadline )); do + if entity_exists 'edge-routers' "$ZITI_ROUTER_NAME"; then + log "router ${ZITI_ROUTER_NAME} registered" + return + fi + log "waiting for router ${ZITI_ROUTER_NAME}" + sleep "$ROUTER_DISCOVERY_INTERVAL" + done + log "router ${ZITI_ROUTER_NAME} not found before timeout" + exit 1 +} + +ensure_router_roles() { + log "ensuring router role attributes for ${ZITI_ROUTER_NAME}" + "${ZITI_BIN}" edge update edge-router "$ZITI_ROUTER_NAME" --role-attributes "$ROUTER_ROLE_ATTRIBUTES" >/dev/null +} + +ensure_service() { + if entity_exists 'services' "$ZITI_SERVICE_NAME"; then + log "updating service attributes for ${ZITI_SERVICE_NAME}" + "${ZITI_BIN}" edge update service "$ZITI_SERVICE_NAME" \ + --role-attributes "$SERVICE_ROLE_ATTRIBUTES" \ + --terminator-strategy smartrouting \ + --encryption ON >/dev/null + return + fi + log "creating service ${ZITI_SERVICE_NAME}" + "${ZITI_BIN}" edge create service "$ZITI_SERVICE_NAME" \ + --role-attributes "$SERVICE_ROLE_ATTRIBUTES" \ + --terminator-strategy smartrouting \ + --encryption ON >/dev/null +} + +ensure_service_policy() { + local name=$1 + local type=$2 + local identity_roles=$3 + local service_roles=$4 + if entity_exists 'service-policies' "$name"; then + log "service policy ${name} already exists" + return + fi + log "creating service policy ${name}" + "${ZITI_BIN}" edge create service-policy "$name" "$type" \ + --semantic AllOf \ + --identity-roles "$identity_roles" \ + --service-roles "$service_roles" >/dev/null +} + +ensure_service_edge_router_policy() { + local name="${ZITI_SERVICE_NAME}.edge-router" + if entity_exists 'service-edge-router-policies' "$name"; then + log "service-edge-router policy ${name} already exists" + return + fi + log "creating service-edge-router policy ${name}" + "${ZITI_BIN}" edge create service-edge-router-policy "$name" \ + --semantic AllOf \ + --edge-router-roles "$ROUTER_ROLES" \ + --service-roles "$SERVICE_ROLES" >/dev/null +} + +ensure_identity_router_policy() { + local name="${ZITI_SERVICE_NAME}.identities.use-router" + if entity_exists 'edge-router-policies' "$name"; then + log "edge-router policy ${name} already exists" + return + fi + log "creating edge-router policy ${name}" + local identity_roles="${PLATFORM_ROLES},${RUNNER_ROLES}" + "${ZITI_BIN}" edge create edge-router-policy "$name" \ + --semantic AnyOf \ + --identity-roles "$identity_roles" \ + --edge-router-roles "$ROUTER_ROLES" >/dev/null +} + +ensure_identity_entity() { + local name=$1 + local role_attributes=$2 + if entity_exists 'identities' "$name"; then + log "updating identity attributes for ${name}" + "${ZITI_BIN}" edge update identity "$name" --role-attributes "$role_attributes" >/dev/null + return + fi + log "creating identity ${name}" + "${ZITI_BIN}" edge create identity "$name" --role-attributes "$role_attributes" >/dev/null +} + +enroll_identity() { + local identity_name=$1 + local destination=$2 + if [[ -s "$destination" ]]; then + log "identity file already present for ${identity_name}" + return + fi + mkdir -p "$(dirname "$destination")" + local jwt_file="${TMP_DIR}/${identity_name}.jwt" + rm -f "$jwt_file" + log "creating enrollment for ${identity_name}" + "${ZITI_BIN}" edge create enrollment ott "$identity_name" \ + --duration "$ENROLLMENT_DURATION_MINUTES" \ + --jwt-output-file "$jwt_file" >/dev/null + log "enrolling identity material for ${identity_name}" + "${ZITI_BIN}" edge enroll "$jwt_file" --out "$destination" --rm >/dev/null + rm -f "$jwt_file" +} + +ensure_identities() { + ensure_identity_entity "$ZITI_PLATFORM_IDENTITY_NAME" "$PLATFORM_ROLE_ATTRIBUTES" + ensure_identity_entity "$ZITI_RUNNER_IDENTITY_NAME" "$RUNNER_ROLE_ATTRIBUTES" + enroll_identity "$ZITI_PLATFORM_IDENTITY_NAME" "$ZITI_PLATFORM_IDENTITY_FILE" + enroll_identity "$ZITI_RUNNER_IDENTITY_NAME" "$ZITI_RUNNER_IDENTITY_FILE" +} + +main() { + log 'starting OpenZiti controller initialization' + source_environment + ensure_access_control_seed + wait_for_router + ensure_router_roles + ensure_service + ensure_service_policy "${ZITI_SERVICE_NAME}.dial" 'Dial' "$PLATFORM_ROLES" "$SERVICE_ROLES" + ensure_service_policy "${ZITI_SERVICE_NAME}.bind" 'Bind' "$RUNNER_ROLES" "$SERVICE_ROLES" + ensure_service_edge_router_policy + ensure_identity_router_policy + ensure_identities + log 'OpenZiti controller initialization completed' +} + +main "$@" From e7d404f6bd872d9fa5c31b0946b913e44180170e Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 12:06:01 +0000 Subject: [PATCH 13/22] fix(ziti): pin controller defaults --- docker-compose.yml | 74 +++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index af7e96fdc..a6f80d042 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -250,23 +250,26 @@ services: ziti-controller: image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} container_name: ziti-controller + hostname: ziti-controller restart: unless-stopped oom_score_adj: -900 entrypoint: - "/var/openziti/scripts/run-controller.sh" environment: - ZITI_CTRL_NAME: ${ZITI_CTRL_NAME:-dev-controller} - ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} - ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} - ZITI_CTRL_EDGE_IP_OVERRIDE: ${ZITI_CTRL_EDGE_IP_OVERRIDE:-127.0.0.1} - ZITI_CTRL_ADVERTISED_PORT: ${ZITI_CTRL_ADVERTISED_PORT:-6262} - ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION: ${ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION:-168h} - ZITI_ROUTER_ENROLLMENT_DURATION: ${ZITI_ROUTER_ENROLLMENT_DURATION:-168h} - ZITI_USER: ${ZITI_USER:-admin} - ZITI_PWD: ${ZITI_PWD:-admin} + ZITI_NETWORK: ${ZITI_NETWORK:-dev.agyn-platform} + ZITI_CTRL_NAME: dev-controller + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: "1280" + ZITI_CTRL_EDGE_IP_OVERRIDE: 127.0.0.1 + ZITI_CTRL_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_ADVERTISED_PORT: "6262" + ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION: 168h + ZITI_ROUTER_ENROLLMENT_DURATION: 168h + ZITI_USER: admin + ZITI_PWD: admin ports: - - "127.0.0.1:${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280}:${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280}" - - "127.0.0.1:${ZITI_CTRL_ADVERTISED_PORT:-6262}:${ZITI_CTRL_ADVERTISED_PORT:-6262}" + - "127.0.0.1:1280:1280" + - "127.0.0.1:6262:6262" healthcheck: test: [ @@ -294,19 +297,20 @@ services: - "/bin/bash" - "/scripts/ziti/controller-init.sh" environment: - ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} - ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} - ZITI_USER: ${ZITI_USER:-admin} - ZITI_PWD: ${ZITI_PWD:-admin} - ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} - ZITI_PLATFORM_IDENTITY_NAME: ${ZITI_PLATFORM_IDENTITY_NAME:-dev.agyn-platform.platform-server} - ZITI_RUNNER_IDENTITY_NAME: ${ZITI_RUNNER_IDENTITY_NAME:-dev.agyn-platform.docker-runner} - ZITI_ROUTER_NAME: ${ZITI_ROUTER_NAME:-dev-edge-router} - ZITI_ENROLLMENT_DURATION_MINUTES: ${ZITI_ENROLLMENT_DURATION_MINUTES:-1440} + ZITI_NETWORK: ${ZITI_NETWORK:-dev.agyn-platform} + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: "1280" + ZITI_USER: admin + ZITI_PWD: admin + ZITI_SERVICE_NAME: dev.agyn-platform.platform-api + ZITI_PLATFORM_IDENTITY_NAME: dev.agyn-platform.platform-server + ZITI_RUNNER_IDENTITY_NAME: dev.agyn-platform.docker-runner + ZITI_ROUTER_NAME: dev-edge-router + ZITI_ENROLLMENT_DURATION_MINUTES: "1440" ZITI_IDENTITIES_DIR: /identities ZITI_IDENTITIES_TMP: /ziti-tmp - ZITI_PLATFORM_IDENTITY_FILE: ${ZITI_PLATFORM_IDENTITY_FILE:-/identities/dev.agyn-platform.platform-server.json} - ZITI_RUNNER_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/identities/dev.agyn-platform.docker-runner.json} + ZITI_PLATFORM_IDENTITY_FILE: /identities/dev.agyn-platform.platform-server.json + ZITI_RUNNER_IDENTITY_FILE: /identities/dev.agyn-platform.docker-runner.json volumes: - type: bind source: ./.ziti/controller @@ -326,6 +330,7 @@ services: ziti-edge-router: image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} container_name: ziti-edge-router + hostname: ziti-edge-router restart: unless-stopped depends_on: ziti-controller: @@ -333,19 +338,20 @@ services: entrypoint: /bin/bash command: "/var/openziti/scripts/run-router.sh edge" environment: - ZITI_CTRL_ADVERTISED_ADDRESS: ${ZITI_CTRL_ADVERTISED_ADDRESS:-ziti-controller} - ZITI_CTRL_ADVERTISED_PORT: ${ZITI_CTRL_ADVERTISED_PORT:-6262} - ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ${ZITI_CTRL_EDGE_ADVERTISED_ADDRESS:-ziti-controller} - ZITI_CTRL_EDGE_ADVERTISED_PORT: ${ZITI_CTRL_EDGE_ADVERTISED_PORT:-1280} - ZITI_ROUTER_NAME: ${ZITI_ROUTER_NAME:-dev-edge-router} - ZITI_ROUTER_ADVERTISED_ADDRESS: ${ZITI_ROUTER_ADVERTISED_ADDRESS:-ziti-edge-router} - ZITI_ROUTER_PORT: ${ZITI_ROUTER_PORT:-3022} - ZITI_ROUTER_LISTENER_BIND_PORT: ${ZITI_ROUTER_LISTENER_BIND_PORT:-10080} - ZITI_ROUTER_ROLES: ${ZITI_ROUTER_ROLES:-public} - ZITI_USER: ${ZITI_USER:-admin} - ZITI_PWD: ${ZITI_PWD:-admin} + ZITI_NETWORK: ${ZITI_NETWORK:-dev.agyn-platform} + ZITI_CTRL_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_ADVERTISED_PORT: "6262" + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: "1280" + ZITI_ROUTER_NAME: dev-edge-router + ZITI_ROUTER_ADVERTISED_ADDRESS: ziti-edge-router + ZITI_ROUTER_PORT: "3022" + ZITI_ROUTER_LISTENER_BIND_PORT: "10080" + ZITI_ROUTER_ROLES: public + ZITI_USER: admin + ZITI_PWD: admin ports: - - "127.0.0.1:${ZITI_ROUTER_PORT:-3022}:${ZITI_ROUTER_PORT:-3022}" + - "127.0.0.1:3022:3022" volumes: - type: bind source: ./.ziti/controller From 367096a980bbdcb6eeed3274b144b64afd5910c4 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 13:08:25 +0000 Subject: [PATCH 14/22] fix(ziti): harden init workflow --- README.md | 11 ++++++++++- docker-compose.yml | 2 +- docs/containers/ziti.md | 24 +++++++++++++++++++++++- scripts/ziti/controller-init.sh | 17 ++++++++++++++--- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 587398e4a..d2124d7fc 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,14 @@ overlay instead of the Docker bridge network: 2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). 3. Start the controller stack: `docker compose up -d ziti-controller ziti-edge-router`. + - Watch `docker compose logs -f ziti-edge-router` until you see the router enroll and connect to `ziti-controller`. + - For a clean bootstrap, stop the stack and wipe any stale state first: + +```bash +docker compose down -v ziti-controller ziti-edge-router +rm -rf ./.ziti/controller ./.ziti/identities ./.ziti/tmp +``` + 4. Bootstrap the controller state (service, policies, identities) via the bundled init job: ```bash @@ -190,7 +198,8 @@ docker compose run --rm ziti-controller-init ``` The init container wraps the OpenZiti CLI, mirrors identity JSON into `./.ziti/identities`, and can be re-run -whenever you need to regenerate enrollment material (no host `ziti` binary required). +whenever you need to regenerate enrollment material (no host `ziti` binary required). If the job reports that the +router has not enrolled yet, keep the `ziti-edge-router` container running, wait for it to connect, then re-run the init job. 5. Launch docker-runner and platform-server normally (either via `pnpm dev` or `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). diff --git a/docker-compose.yml b/docker-compose.yml index a6f80d042..7c8866243 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -347,7 +347,7 @@ services: ZITI_ROUTER_ADVERTISED_ADDRESS: ziti-edge-router ZITI_ROUTER_PORT: "3022" ZITI_ROUTER_LISTENER_BIND_PORT: "10080" - ZITI_ROUTER_ROLES: public + ZITI_ROUTER_ROLES: router.platform ZITI_USER: admin ZITI_PWD: admin ports: diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index d7972da6c..2c3645ea9 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -26,7 +26,7 @@ pnpm approve-builds > pnpm --dir node_modules/.pnpm/@openziti+ziti-sdk-nodejs@0.27.0/node_modules/@openziti/ziti-sdk-nodejs run install > ``` -2. Ensure the OpenZiti controller stack is running: +2. Ensure the OpenZiti controller stack is running and the router finishes enrolling: ```bash docker compose up -d ziti-controller ziti-edge-router @@ -36,12 +36,20 @@ docker compose up -d ziti-controller ziti-edge-router > start them with `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` > so they share the same controller and network. +Watch `docker compose logs -f ziti-edge-router` until you see the router enroll ("successfully connected to controller") +before attempting the init job. If the router refuses to start, wipe `.ziti/controller` using the reset steps below and +retry. + 3. Bootstrap the controller via the init job (idempotent; re-run whenever identity JSON needs to be regenerated): ```bash docker compose run --rm ziti-controller-init ``` +The job now waits up to five minutes for the router named by `ZITI_ROUTER_NAME` (default `dev-edge-router`). If the +router is still missing after the timeout it logs a warning, skips router role updates, and exits so you can rerun it +later. Set `ZITI_SKIP_ROUTER_WAIT=true` to disable the wait entirely. + 4. Copy `.env` files and enable OpenZiti flags. Use the template that matches how you run the services: ### Host (`pnpm dev`) @@ -137,3 +145,17 @@ Seeing the readiness log after step 1 indicates the end-to-end tunnel is operati > To reset the environment delete `./.ziti/identities` and `./.ziti/tmp` (or the `/opt/app/.ziti/*` mounts inside Docker), then restart the stack so the platform-server > can re-enroll identities. + +## Resetting OpenZiti state + +The controller stores PKI and router enrollment artifacts in `./.ziti/controller`. Stale files can cause hostname or TLS +conflicts, so a clean bootstrap consists of: + +```bash +docker compose down -v ziti-controller ziti-edge-router +rm -rf ./.ziti/controller ./.ziti/identities ./.ziti/tmp +docker compose up -d ziti-controller ziti-edge-router +docker compose run --rm ziti-controller-init +``` + +Re-run the init job whenever it warns that the router has not registered so role attributes stay in sync. diff --git a/scripts/ziti/controller-init.sh b/scripts/ziti/controller-init.sh index e190dcede..00df31f1b 100755 --- a/scripts/ziti/controller-init.sh +++ b/scripts/ziti/controller-init.sh @@ -9,9 +9,10 @@ ZITI_SCRIPTS=${ZITI_SCRIPTS:-/var/openziti/scripts} ZITI_HOME=${ZITI_HOME:-/persistent} IDENTITIES_DIR=${ZITI_IDENTITIES_DIR:-/identities} TMP_DIR=${ZITI_IDENTITIES_TMP:-/ziti-tmp} -ROUTER_DISCOVERY_TIMEOUT=${ZITI_ROUTER_DISCOVERY_TIMEOUT_SECONDS:-120} +ROUTER_DISCOVERY_TIMEOUT=${ZITI_ROUTER_DISCOVERY_TIMEOUT_SECONDS:-300} ROUTER_DISCOVERY_INTERVAL=${ZITI_ROUTER_DISCOVERY_INTERVAL_SECONDS:-2} ENROLLMENT_DURATION_MINUTES=${ZITI_ENROLLMENT_DURATION_MINUTES:-1440} +ROUTER_PRESENT=false APP_ATTRIBUTE='app.agyn-platform' SERVICE_ATTRIBUTE='service.platform-api' @@ -87,20 +88,30 @@ ensure_access_control_seed() { } wait_for_router() { + if [[ "${ZITI_SKIP_ROUTER_WAIT:-false}" == "true" ]]; then + log 'router wait disabled via ZITI_SKIP_ROUTER_WAIT' + return + fi local deadline=$(( $(date +%s) + ROUTER_DISCOVERY_TIMEOUT )) + log "waiting for router ${ZITI_ROUTER_NAME} to register (timeout ${ROUTER_DISCOVERY_TIMEOUT}s)" while (( $(date +%s) < deadline )); do if entity_exists 'edge-routers' "$ZITI_ROUTER_NAME"; then log "router ${ZITI_ROUTER_NAME} registered" + ROUTER_PRESENT=true return fi log "waiting for router ${ZITI_ROUTER_NAME}" sleep "$ROUTER_DISCOVERY_INTERVAL" done - log "router ${ZITI_ROUTER_NAME} not found before timeout" - exit 1 + log "router ${ZITI_ROUTER_NAME} not found before timeout — skipping router-specific updates." + log "Ensure 'docker compose up -d ziti-edge-router' succeeded, then re-run this init job once the router enrolls." } ensure_router_roles() { + if [[ "$ROUTER_PRESENT" != true ]]; then + log "router ${ZITI_ROUTER_NAME} not registered yet; skipping router role assignment" + return + fi log "ensuring router role attributes for ${ZITI_ROUTER_NAME}" "${ZITI_BIN}" edge update edge-router "$ZITI_ROUTER_NAME" --role-attributes "$ROUTER_ROLE_ATTRIBUTES" >/dev/null } From b5ecef4c16e9ae2f7bfedc220727a42ce6e1985b Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 14:33:54 +0000 Subject: [PATCH 15/22] fix(openziti): stabilize docker runner stack --- docker-compose.dev.yml | 2 + docker-compose.yml | 2 + packages/docker-runner/Dockerfile | 10 +++-- packages/docker-runner/src/contracts/api.ts | 2 +- packages/docker-runner/src/contracts/auth.ts | 2 +- packages/docker-runner/src/index.ts | 22 +++++----- .../docker-runner/src/lib/container.handle.ts | 4 +- .../src/lib/container.service.ts | 14 +++---- .../src/lib/containerRegistry.port.ts | 2 +- .../src/lib/dockerClient.port.ts | 4 +- packages/docker-runner/src/service/app.ts | 8 ++-- packages/docker-runner/src/service/main.ts | 8 ++-- scripts/ziti/controller-init.sh | 42 +++++++++++++++---- 13 files changed, 77 insertions(+), 45 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 98df91635..721694ac0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -38,6 +38,7 @@ services: ZITI_RUNNER_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} ZITI_IDENTITIES_DIR: ${ZITI_IDENTITIES_DIR:-/opt/app/.ziti/identities} ZITI_TMP_DIR: ${ZITI_TMP_DIR:-/opt/app/.ziti/tmp} + GRAPH_REPO_PATH: ${GRAPH_REPO_PATH:-/opt/app/data/graph} ports: - "${PLATFORM_SERVER_PORT:-3010}:3010" - "${ZITI_RUNNER_PROXY_PORT:-17071}:17071" @@ -55,6 +56,7 @@ services: build: context: . dockerfile: packages/docker-runner/Dockerfile + user: root restart: unless-stopped depends_on: ziti-edge-router: diff --git a/docker-compose.yml b/docker-compose.yml index 7c8866243..67c5e96e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,8 @@ services: interval: 10s timeout: 5s retries: 5 + networks: + - agents_net vault: image: hashicorp/vault:1.17 diff --git a/packages/docker-runner/Dockerfile b/packages/docker-runner/Dockerfile index 92c0d392d..657b1acb3 100644 --- a/packages/docker-runner/Dockerfile +++ b/packages/docker-runner/Dockerfile @@ -23,7 +23,10 @@ FROM base AS build COPY . . -RUN pnpm install --filter @agyn/docker-runner... --offline --frozen-lockfile +RUN pnpm install \ + --filter @agyn/docker-runner... \ + --filter @agyn/platform-server \ + --offline --frozen-lockfile RUN pnpm --filter @agyn/docker-runner run build @@ -32,9 +35,10 @@ RUN pnpm deploy --filter @agyn/docker-runner --prod --legacy /opt/app FROM node:20-slim AS runtime ENV NODE_ENV=production \ - PORT=7071 + PORT=7071 \ + NODE_OPTIONS=--experimental-specifier-resolution=node -WORKDIR /opt/app/packages/docker-runner +WORKDIR /opt/app RUN apt-get update \ && apt-get install -y --no-install-recommends git \ diff --git a/packages/docker-runner/src/contracts/api.ts b/packages/docker-runner/src/contracts/api.ts index 42eeef9e9..cb975ebcb 100644 --- a/packages/docker-runner/src/contracts/api.ts +++ b/packages/docker-runner/src/contracts/api.ts @@ -1,5 +1,5 @@ import type Docker from 'dockerode'; -import type { ContainerOpts, ExecOptions, ExecResult, Platform } from '../lib/types'; +import type { ContainerOpts, ExecOptions, ExecResult, Platform } from '../lib/types.js'; export type ErrorPayload = { error: { diff --git a/packages/docker-runner/src/contracts/auth.ts b/packages/docker-runner/src/contracts/auth.ts index 8dbebc5e0..e6d61518a 100644 --- a/packages/docker-runner/src/contracts/auth.ts +++ b/packages/docker-runner/src/contracts/auth.ts @@ -1,5 +1,5 @@ import crypto from 'node:crypto'; -import { canonicalJsonStringify } from './json'; +import { canonicalJsonStringify } from './json.js'; export type SignatureHeaders = { timestamp: string; diff --git a/packages/docker-runner/src/index.ts b/packages/docker-runner/src/index.ts index 44a37adc5..d4e53ec20 100644 --- a/packages/docker-runner/src/index.ts +++ b/packages/docker-runner/src/index.ts @@ -1,11 +1,11 @@ -export * from './lib/container.service'; -export * from './lib/container.handle'; -export * from './lib/container.mounts'; -export * from './lib/containerStream.util'; -export * from './lib/containerRegistry.port'; -export * from './lib/dockerClient.port'; -export * from './lib/execTimeout'; -export * from './lib/types'; -export * from './contracts/auth'; -export * from './contracts/json'; -export * from './contracts/api'; +export * from './lib/container.service.js'; +export * from './lib/container.handle.js'; +export * from './lib/container.mounts.js'; +export * from './lib/containerStream.util.js'; +export * from './lib/containerRegistry.port.js'; +export * from './lib/dockerClient.port.js'; +export * from './lib/execTimeout.js'; +export * from './lib/types.js'; +export * from './contracts/auth.js'; +export * from './contracts/json.js'; +export * from './contracts/api.js'; diff --git a/packages/docker-runner/src/lib/container.handle.ts b/packages/docker-runner/src/lib/container.handle.ts index b0964b2fa..be21f7b2e 100644 --- a/packages/docker-runner/src/lib/container.handle.ts +++ b/packages/docker-runner/src/lib/container.handle.ts @@ -1,5 +1,5 @@ -import type { DockerClientPort } from './dockerClient.port'; -import type { ExecOptions } from './types'; +import type { DockerClientPort } from './dockerClient.port.js'; +import type { ExecOptions } from './types.js'; /** * Lightweight entity wrapper representing a running (or created) container. diff --git a/packages/docker-runner/src/lib/container.service.ts b/packages/docker-runner/src/lib/container.service.ts index d2682419d..cd42b47a5 100644 --- a/packages/docker-runner/src/lib/container.service.ts +++ b/packages/docker-runner/src/lib/container.service.ts @@ -1,12 +1,12 @@ import { Injectable, Logger } from '@nestjs/common'; import Docker, { ContainerCreateOptions, Exec, type GetEventsOptions } from 'dockerode'; import { PassThrough, Writable } from 'node:stream'; -import { ContainerHandle } from './container.handle'; -import { mapInspectMounts } from './container.mounts'; -import { createUtf8Collector, demuxDockerMultiplex } from './containerStream.util'; -import { ExecIdleTimeoutError, ExecTimeoutError, isExecIdleTimeoutError, isExecTimeoutError } from './execTimeout'; -import type { ContainerRegistryPort } from './containerRegistry.port'; -import type { DockerClientPort } from './dockerClient.port'; +import { ContainerHandle } from './container.handle.js'; +import { mapInspectMounts } from './container.mounts.js'; +import { createUtf8Collector, demuxDockerMultiplex } from './containerStream.util.js'; +import { ExecIdleTimeoutError, ExecTimeoutError, isExecIdleTimeoutError, isExecTimeoutError } from './execTimeout.js'; +import type { ContainerRegistryPort } from './containerRegistry.port.js'; +import type { DockerClientPort } from './dockerClient.port.js'; import { ContainerOpts, ExecOptions, @@ -17,7 +17,7 @@ import { LogsStreamSession, Platform, PLATFORM_LABEL, -} from './types'; +} from './types.js'; const INTERACTIVE_EXEC_CLOSE_CAPTURE_LIMIT = 256 * 1024; // 256 KiB of characters (~512 KiB memory) diff --git a/packages/docker-runner/src/lib/containerRegistry.port.ts b/packages/docker-runner/src/lib/containerRegistry.port.ts index a0490a7f9..0250c6adf 100644 --- a/packages/docker-runner/src/lib/containerRegistry.port.ts +++ b/packages/docker-runner/src/lib/containerRegistry.port.ts @@ -1,4 +1,4 @@ -import type { ContainerMount } from './container.mounts'; +import type { ContainerMount } from './container.mounts.js'; export type RegisterContainerStartInput = { containerId: string; diff --git a/packages/docker-runner/src/lib/dockerClient.port.ts b/packages/docker-runner/src/lib/dockerClient.port.ts index 66fd1be98..29d64978e 100644 --- a/packages/docker-runner/src/lib/dockerClient.port.ts +++ b/packages/docker-runner/src/lib/dockerClient.port.ts @@ -1,4 +1,4 @@ -import type { ContainerHandle } from './container.handle'; +import type { ContainerHandle } from './container.handle.js'; import type { ContainerOpts, ExecOptions, @@ -8,7 +8,7 @@ import type { LogsStreamOptions, LogsStreamSession, Platform, -} from './types'; +} from './types.js'; export type DockerEventFilters = Record>; diff --git a/packages/docker-runner/src/service/app.ts b/packages/docker-runner/src/service/app.ts index 209bb5dbd..56e270ed1 100644 --- a/packages/docker-runner/src/service/app.ts +++ b/packages/docker-runner/src/service/app.ts @@ -10,10 +10,10 @@ import { verifyAuthHeaders, type ContainerOpts, type ExecOptions, -} from '..'; -import { createDockerEventsParser } from './dockerEvents.parser'; -import type { RunnerConfig } from './config'; -import { closeWebsocket, getWebsocket, type SocketStream } from './websocket.util'; +} from '../index.js'; +import { createDockerEventsParser } from './dockerEvents.parser.js'; +import type { RunnerConfig } from './config.js'; +import { closeWebsocket, getWebsocket, type SocketStream } from './websocket.util.js'; const ensureImageSchema = z.object({ image: z.string().min(1), diff --git a/packages/docker-runner/src/service/main.ts b/packages/docker-runner/src/service/main.ts index 03e6462d9..0482e6a3e 100644 --- a/packages/docker-runner/src/service/main.ts +++ b/packages/docker-runner/src/service/main.ts @@ -1,10 +1,10 @@ -import './env'; +import './env.js'; import type { FastifyInstance } from 'fastify'; -import { loadRunnerConfig } from './config'; -import { createRunnerApp } from './app'; -import { startZitiIngress } from './ziti.ingress'; +import { loadRunnerConfig } from './config.js'; +import { createRunnerApp } from './app.js'; +import { startZitiIngress } from './ziti.ingress.js'; async function bootstrap(): Promise { let app: FastifyInstance | undefined; diff --git a/scripts/ziti/controller-init.sh b/scripts/ziti/controller-init.sh index 00df31f1b..adfba7a23 100755 --- a/scripts/ziti/controller-init.sh +++ b/scripts/ziti/controller-init.sh @@ -73,7 +73,15 @@ entity_exists() { if ! output=$("${ZITI_BIN}" edge list "$resource" "$filter" -j 2>/dev/null); then return 1 fi - printf '%s' "$output" | grep -F "\"name\":\"${name}\"" >/dev/null + printf '%s' "$output" | jq -e --arg name "$name" 'any((.data // [])[]; .name == $name)' >/dev/null 2>&1 +} + +fetch_pending_identity_jwt() { + local name=$1 + local filter + filter=$(printf 'name = "%s"' "$name") + "${ZITI_BIN}" edge list identities "$filter" -j 2>/dev/null \ + | jq -r --arg name "$name" '((.data // [])[] | select(.name == $name) | .enrollment.ott.jwt // empty)' } ensure_access_control_seed() { @@ -178,13 +186,17 @@ ensure_identity_router_policy() { ensure_identity_entity() { local name=$1 local role_attributes=$2 + local enrollment_seed=${3:-${TMP_DIR}/${name}.jwt} if entity_exists 'identities' "$name"; then log "updating identity attributes for ${name}" "${ZITI_BIN}" edge update identity "$name" --role-attributes "$role_attributes" >/dev/null + rm -f "$enrollment_seed" return fi log "creating identity ${name}" - "${ZITI_BIN}" edge create identity "$name" --role-attributes "$role_attributes" >/dev/null + "${ZITI_BIN}" edge create identity "$name" \ + --role-attributes "$role_attributes" \ + --jwt-output-file "$enrollment_seed" >/dev/null } enroll_identity() { @@ -196,19 +208,31 @@ enroll_identity() { fi mkdir -p "$(dirname "$destination")" local jwt_file="${TMP_DIR}/${identity_name}.jwt" - rm -f "$jwt_file" - log "creating enrollment for ${identity_name}" - "${ZITI_BIN}" edge create enrollment ott "$identity_name" \ - --duration "$ENROLLMENT_DURATION_MINUTES" \ - --jwt-output-file "$jwt_file" >/dev/null + if [[ ! -s "$jwt_file" ]]; then + local pending_jwt + pending_jwt=$(fetch_pending_identity_jwt "$identity_name" || true) + if [[ -n "$pending_jwt" ]]; then + printf '%s' "$pending_jwt" >"$jwt_file" + else + log "creating enrollment for ${identity_name}" + if ! "${ZITI_BIN}" edge create enrollment ott "$identity_name" \ + --duration "$ENROLLMENT_DURATION_MINUTES" \ + --jwt-output-file "$jwt_file" >/dev/null; then + log "failed to create enrollment for ${identity_name}; see controller logs for details" + return 1 + fi + fi + fi log "enrolling identity material for ${identity_name}" "${ZITI_BIN}" edge enroll "$jwt_file" --out "$destination" --rm >/dev/null rm -f "$jwt_file" } ensure_identities() { - ensure_identity_entity "$ZITI_PLATFORM_IDENTITY_NAME" "$PLATFORM_ROLE_ATTRIBUTES" - ensure_identity_entity "$ZITI_RUNNER_IDENTITY_NAME" "$RUNNER_ROLE_ATTRIBUTES" + local platform_jwt="${TMP_DIR}/${ZITI_PLATFORM_IDENTITY_NAME}.jwt" + local runner_jwt="${TMP_DIR}/${ZITI_RUNNER_IDENTITY_NAME}.jwt" + ensure_identity_entity "$ZITI_PLATFORM_IDENTITY_NAME" "$PLATFORM_ROLE_ATTRIBUTES" "$platform_jwt" + ensure_identity_entity "$ZITI_RUNNER_IDENTITY_NAME" "$RUNNER_ROLE_ATTRIBUTES" "$runner_jwt" enroll_identity "$ZITI_PLATFORM_IDENTITY_NAME" "$ZITI_PLATFORM_IDENTITY_FILE" enroll_identity "$ZITI_RUNNER_IDENTITY_NAME" "$ZITI_RUNNER_IDENTITY_FILE" } From a5f3b58d9a7c0eef8f0b4e83cbcfa0bed3c31854 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 14:43:45 +0000 Subject: [PATCH 16/22] fix(docker-runner): align runtime workdir --- packages/docker-runner/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/docker-runner/Dockerfile b/packages/docker-runner/Dockerfile index 657b1acb3..83eab6b60 100644 --- a/packages/docker-runner/Dockerfile +++ b/packages/docker-runner/Dockerfile @@ -30,7 +30,7 @@ RUN pnpm install \ RUN pnpm --filter @agyn/docker-runner run build -RUN pnpm deploy --filter @agyn/docker-runner --prod --legacy /opt/app +RUN pnpm deploy --filter @agyn/docker-runner --prod --legacy /opt/app/packages/docker-runner FROM node:20-slim AS runtime @@ -38,7 +38,7 @@ ENV NODE_ENV=production \ PORT=7071 \ NODE_OPTIONS=--experimental-specifier-resolution=node -WORKDIR /opt/app +WORKDIR /opt/app/packages/docker-runner RUN apt-get update \ && apt-get install -y --no-install-recommends git \ From cda4418e3bfebb638c9eb295091ccde5ef52e16c Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 20:33:23 +0000 Subject: [PATCH 17/22] test(platform-server): stabilize litellm admin integration --- .../litellm.admin.integration.test.ts | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/platform-server/__tests__/litellm.admin.integration.test.ts b/packages/platform-server/__tests__/litellm.admin.integration.test.ts index 140ce25d7..da6f69152 100644 --- a/packages/platform-server/__tests__/litellm.admin.integration.test.ts +++ b/packages/platform-server/__tests__/litellm.admin.integration.test.ts @@ -5,22 +5,23 @@ import { LLMSettingsService } from '../src/settings/llm/llmSettings.service'; import { ConfigService, configSchema } from '../src/core/services/config.service'; import { runnerConfigDefaults } from './helpers/config'; -const LITELLM_BASE = 'http://127.0.0.1:4000'; +const DEFAULT_LITELLM_BASE = 'http://127.0.0.1:4000'; const MASTER_KEY = 'sk-dev-master-1234'; -const createConfig = () => +const createConfig = (baseUrl = DEFAULT_LITELLM_BASE) => new ConfigService().init( configSchema.parse({ llmProvider: 'litellm', - litellmBaseUrl: LITELLM_BASE, + litellmBaseUrl: baseUrl, litellmMasterKey: MASTER_KEY, agentsDatabaseUrl: 'postgres://dev:dev@localhost:5432/agents', ...runnerConfigDefaults, }), ); -function createLiteLLMStubServer(masterKey: string, port = 4000) { +function createLiteLLMStubServer(masterKey: string, port?: number) { const fastify = Fastify({ logger: false }); + let resolvedPort: number | null = port ?? null; type CredentialRecord = { credential_info: Record; @@ -228,11 +229,27 @@ function createLiteLLMStubServer(masterKey: string, port = 4000) { }); const start = async () => { - await fastify.listen({ port, host: '127.0.0.1' }); + await fastify.listen({ port: port ?? 0, host: '127.0.0.1' }); + const address = fastify.server.address(); + if (address && typeof address === 'object') { + resolvedPort = address.port; + return `http://${address.address}:${address.port}`; + } + if (typeof address === 'string') { + const url = address.startsWith('http') ? address : `http://${address}`; + const parsed = new URL(url); + resolvedPort = Number(parsed.port); + return `http://${parsed.hostname}:${parsed.port}`; + } + if (resolvedPort) { + return `http://127.0.0.1:${resolvedPort}`; + } + throw new Error('Failed to determine LiteLLM stub address'); }; const stop = async () => { await fastify.close(); + resolvedPort = null; }; const reset = () => { @@ -240,19 +257,32 @@ function createLiteLLMStubServer(masterKey: string, port = 4000) { models.clear(); }; - return { start, stop, reset, server: fastify } satisfies { - start: () => Promise; + return { + start, + stop, + reset, + server: fastify, + getBaseUrl: () => { + if (resolvedPort === null) { + throw new Error('LiteLLM stub has not been started'); + } + return `http://127.0.0.1:${resolvedPort}`; + }, + } satisfies { + start: () => Promise; stop: () => Promise; reset: () => void; server: FastifyInstance; + getBaseUrl: () => string; }; } describe.sequential('LiteLLM admin integration', () => { - const stub = createLiteLLMStubServer(MASTER_KEY, 4000); + const stub = createLiteLLMStubServer(MASTER_KEY); + let litellmBase = DEFAULT_LITELLM_BASE; beforeAll(async () => { - await stub.start(); + litellmBase = await stub.start(); }, 120_000); afterAll(async () => { @@ -264,17 +294,17 @@ describe.sequential('LiteLLM admin integration', () => { }); it('reports admin status as reachable when LiteLLM responds', async () => { - const service = new LLMSettingsService(createConfig()); + const service = new LLMSettingsService(createConfig(litellmBase)); const status = await service.getAdminStatus(); expect(status).toMatchObject({ configured: true, adminReachable: true, - baseUrl: LITELLM_BASE, + baseUrl: litellmBase, }); }); it('manages credentials and models end-to-end', async () => { - const service = new LLMSettingsService(createConfig()); + const service = new LLMSettingsService(createConfig(litellmBase)); const credentialName = `integration-cred-${Date.now()}`; const modelName = `integration-model-${Date.now()}`; @@ -310,7 +340,7 @@ describe.sequential('LiteLLM admin integration', () => { const health = await service.testModel({ id: modelName }); expect(health).toBeTruthy(); - const runtimeRes = await fetch(`${LITELLM_BASE}/v1/chat/completions`, { + const runtimeRes = await fetch(`${litellmBase}/v1/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${MASTER_KEY}`, From ec2c5497f4e56b2bea498a5c89ee397f554570c6 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Fri, 20 Feb 2026 21:57:48 +0000 Subject: [PATCH 18/22] feat(ziti): improve host-mode bootstrap reliability --- README.md | 21 ++-- docker-compose.yml | 10 ++ docs/containers/ziti.md | 53 ++++++++++- package.json | 3 +- .../dockerRunnerConnectivity.probe.ts | 95 +++++++++++++++---- scripts/ziti/prepare-volumes.sh | 24 +++++ 6 files changed, 178 insertions(+), 28 deletions(-) create mode 100755 scripts/ziti/prepare-volumes.sh diff --git a/README.md b/README.md index d2124d7fc..d2500766a 100644 --- a/README.md +++ b/README.md @@ -179,10 +179,12 @@ docker run --rm -p 8080:80 \ The dev stack now ships an OpenZiti controller, initializer, and edge router. To route docker-runner traffic through the overlay instead of the Docker bridge network: -1. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). -2. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and +1. Prepare the shared volumes once per checkout (`pnpm ziti:prepare`). This keeps `.ziti/controller`, + `.ziti/identities`, and `.ziti/tmp` writable even on SELinux hosts. +2. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). +3. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). -3. Start the controller stack: `docker compose up -d ziti-controller ziti-edge-router`. +4. Start the controller stack: `docker compose up -d ziti-controller ziti-edge-router`. - Watch `docker compose logs -f ziti-edge-router` until you see the router enroll and connect to `ziti-controller`. - For a clean bootstrap, stop the stack and wipe any stale state first: @@ -191,17 +193,18 @@ docker compose down -v ziti-controller ziti-edge-router rm -rf ./.ziti/controller ./.ziti/identities ./.ziti/tmp ``` -4. Bootstrap the controller state (service, policies, identities) via the bundled init job: +5. Bootstrap the controller state (service, policies, identities) via the bundled init job. Passing your UID/GID keeps + the emitted identity files readable without a manual `chmod`: ```bash -docker compose run --rm ziti-controller-init +docker compose run --rm --user "$(id -u):$(id -g)" ziti-controller-init ``` The init container wraps the OpenZiti CLI, mirrors identity JSON into `./.ziti/identities`, and can be re-run whenever you need to regenerate enrollment material (no host `ziti` binary required). If the job reports that the router has not enrolled yet, keep the `ziti-edge-router` container running, wait for it to connect, then re-run the init job. -5. Launch docker-runner and platform-server normally (either via `pnpm dev` or +6. Launch docker-runner and platform-server normally (either via `pnpm dev` or `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). - Host `pnpm` dev keeps the ConfigService defaults, so the proxy binds to `127.0.0.1:17071` unless you override `ZITI_RUNNER_PROXY_HOST`. @@ -209,7 +212,11 @@ router has not enrolled yet, keep the `ziti-edge-router` container running, wait the host. In both cases traffic is accessible from the host via `127.0.0.1:17071` once the proxy reports ready. -See [docs/containers/ziti.md](docs/containers/ziti.md) for the full walkthrough and smoke test commands. +The platform-server now retries the connectivity probe (defaults: 30 attempts, 2s interval). Override via +`DOCKER_RUNNER_PROBE_MAX_ATTEMPTS` / `DOCKER_RUNNER_PROBE_INTERVAL_MS` if you need a longer window before the runner +comes online. + +See [docs/containers/ziti.md](docs/containers/ziti.md) for the step-by-step host-mode workflow and smoke test commands. ## Configuration diff --git a/docker-compose.yml b/docker-compose.yml index 67c5e96e7..723ac5d1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -285,6 +285,8 @@ services: - type: bind source: ./.ziti/controller target: /persistent + bind: + selinux: z networks: - agents_net @@ -317,12 +319,18 @@ services: - type: bind source: ./.ziti/controller target: /persistent + bind: + selinux: z - type: bind source: ./.ziti/identities target: /identities + bind: + selinux: z - type: bind source: ./.ziti/tmp target: /ziti-tmp + bind: + selinux: z - type: bind source: ./scripts/ziti target: /scripts/ziti @@ -358,6 +366,8 @@ services: - type: bind source: ./.ziti/controller target: /persistent + bind: + selinux: z networks: - agents_net diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 2c3645ea9..9ef53c6b0 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -13,7 +13,13 @@ bridge network. ## Prerequisites -1. Install dependencies and explicitly allow the OpenZiti SDK build step (pnpm blocks install scripts by default): +1. Prepare the local `.ziti` directory (creates controller/identity/tmp folders with permissive permissions and SELinux labels when available): + +```bash +pnpm ziti:prepare +``` + +2. Install dependencies and explicitly allow the OpenZiti SDK build step (pnpm blocks install scripts by default): ```bash pnpm approve-builds @@ -26,7 +32,7 @@ pnpm approve-builds > pnpm --dir node_modules/.pnpm/@openziti+ziti-sdk-nodejs@0.27.0/node_modules/@openziti/ziti-sdk-nodejs run install > ``` -2. Ensure the OpenZiti controller stack is running and the router finishes enrolling: +3. Ensure the OpenZiti controller stack is running and the router finishes enrolling: ```bash docker compose up -d ziti-controller ziti-edge-router @@ -40,10 +46,10 @@ Watch `docker compose logs -f ziti-edge-router` until you see the router enroll before attempting the init job. If the router refuses to start, wipe `.ziti/controller` using the reset steps below and retry. -3. Bootstrap the controller via the init job (idempotent; re-run whenever identity JSON needs to be regenerated): +4. Bootstrap the controller via the init job (idempotent; re-run whenever identity JSON needs to be regenerated). Running the job with your host UID/GID keeps the generated identity files readable without extra chmod steps: ```bash -docker compose run --rm ziti-controller-init +docker compose run --rm --user "$(id -u):$(id -g)" ziti-controller-init ``` The job now waits up to five minutes for the router named by `ZITI_ROUTER_NAME` (default `dev-edge-router`). If the @@ -106,6 +112,45 @@ ZITI_TMP_DIR=/opt/app/.ziti/tmp ZITI_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json ``` +## Host-mode workflow + +After completing the prerequisites and enabling the `.env` entries above, the developer stack can be verified on the host with the following sequence (clean-room friendly): + +1. Bring up the persistence dependencies: + +```bash +docker compose up -d postgres agents-db litellm-db litellm +``` + +2. Start the docker-runner in a terminal (requires `ZITI_ENABLED=true`). Wait for both the Fastify log and the Ziti ingress message: + +```bash +pnpm --filter @agyn/docker-runner dev +# ... +# {"level":30,..."msg":"Server listening at http://127.0.0.1:7071"} +# Ziti ingress ready for service dev.agyn-platform.platform-api +``` + +3. Start the platform-server in a separate terminal: + +```bash +pnpm --filter @agyn/platform-server dev +``` + +The `DockerRunnerConnectivityProbe` now waits for the local Ziti proxy before giving up. It retries 30 times with a 2s interval by default (~60s). Override the timing via `DOCKER_RUNNER_PROBE_MAX_ATTEMPTS` and `DOCKER_RUNNER_PROBE_INTERVAL_MS` if you need longer windows for slower machines. + +4. Validate the tunnel: + +```bash +curl http://127.0.0.1:17071/v1/ready +``` + +Expected response: + +```json +{"status":"ready"} +``` + ## Runtime flow 1. `docker compose run --rm ziti-controller-init` wraps the OpenZiti CLI to create/update the service, diff --git a/package.json b/package.json index 912f33444..ce075cb72 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "test": "pnpm -r --workspace-concurrency=1 run --if-present test", "convert-graphs": "pnpm --filter @agyn/graph-converter exec graph-converter", "postinstall": "pnpm -r --if-present run prepare", - "deps:up:podman": "podman compose up -d" + "deps:up:podman": "podman compose up -d", + "ziti:prepare": "bash ./scripts/ziti/prepare-volumes.sh" }, "keywords": [], "author": "", diff --git a/packages/platform-server/src/infra/container/dockerRunnerConnectivity.probe.ts b/packages/platform-server/src/infra/container/dockerRunnerConnectivity.probe.ts index d1fc7bf15..499a842a7 100644 --- a/packages/platform-server/src/infra/container/dockerRunnerConnectivity.probe.ts +++ b/packages/platform-server/src/infra/container/dockerRunnerConnectivity.probe.ts @@ -1,3 +1,5 @@ +import { setTimeout as delay } from 'node:timers/promises'; + import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '../../core/services/config.service'; @@ -33,26 +35,87 @@ export class DockerRunnerConnectivityProbe implements OnModuleInit { return; } - try { - const response = await this.dockerClient.checkConnectivity(); - this.logger.log('Docker runner connectivity established', { baseUrl, status: response.status }); - } catch (error) { - const payload: Record = { baseUrl }; - if (error instanceof DockerRunnerRequestError) { - payload.statusCode = error.statusCode; - payload.runnerErrorCode = error.errorCode; - payload.retryable = error.retryable; - payload.message = error.message; - } else if (error instanceof Error) { - payload.message = error.message; - } else { - payload.error = error; + await this.probe(baseUrl, this.dockerClient); + } + + private async probe(baseUrl: string, client: HttpDockerRunnerClient): Promise { + const maxAttempts = this.parsePositiveInt( + process.env.DOCKER_RUNNER_PROBE_MAX_ATTEMPTS, + 30, + ); + const intervalMs = this.parsePositiveInt( + process.env.DOCKER_RUNNER_PROBE_INTERVAL_MS, + 2_000, + ); + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + const response = await client.checkConnectivity(); + this.logger.log('Docker runner connectivity established', { + baseUrl, + status: response.status, + attempt, + }); + return; + } catch (error: unknown) { + const payload = this.buildErrorPayload(baseUrl, error); + payload.attempt = attempt; + payload.maxAttempts = maxAttempts; + if (!Object.prototype.hasOwnProperty.call(payload, 'retryable')) { + payload.retryable = this.isRetryable(error); + } + + const shouldRetry = payload.retryable && attempt < maxAttempts; + if (!shouldRetry) { + this.logger.error( + 'Docker runner connectivity check failed', + error instanceof Error ? error.stack : undefined, + payload, + ); + throw error instanceof Error ? error : new Error('docker_runner_connectivity_failed'); + } + + this.logger.warn('Docker runner connectivity not ready; retrying', payload); + await delay(intervalMs); } - this.logger.error('Docker runner connectivity check failed', error instanceof Error ? error.stack : undefined, payload); - throw error; } } + private buildErrorPayload(baseUrl: string, error: unknown): Record { + const payload: Record = { baseUrl }; + if (error instanceof DockerRunnerRequestError) { + payload.statusCode = error.statusCode; + payload.runnerErrorCode = error.errorCode; + payload.retryable = error.retryable; + payload.message = error.message; + return payload; + } + if (error instanceof Error) { + payload.message = error.message; + return payload; + } + payload.error = error; + return payload; + } + + private isRetryable(error: unknown): boolean { + if (error instanceof DockerRunnerRequestError) { + return error.retryable; + } + return true; + } + + private parsePositiveInt(value: string | undefined, fallback: number): number { + if (!value) { + return fallback; + } + const parsed = Number.parseInt(value, 10); + if (Number.isNaN(parsed) || parsed <= 0) { + return fallback; + } + return parsed; + } + private resolveConfig(): ConfigService | undefined { if (this.config) { return this.config; diff --git a/scripts/ziti/prepare-volumes.sh b/scripts/ziti/prepare-volumes.sh new file mode 100755 index 000000000..f3862e479 --- /dev/null +++ b/scripts/ziti/prepare-volumes.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +log() { + printf '[ziti-prepare] %s\n' "$1" +} + +ROOT_DIR=${1:-$(pwd)} +ZITI_DIR="${ROOT_DIR}/.ziti" +CONTROLLER_DIR="${ZITI_DIR}/controller" +IDENTITIES_DIR="${ZITI_DIR}/identities" +TMP_DIR="${ZITI_DIR}/tmp" + +mkdir -p "$CONTROLLER_DIR" "$IDENTITIES_DIR" "$TMP_DIR" + +log "Ensuring writable permissions under ${ZITI_DIR}" +chmod -R 0777 "$ZITI_DIR" + +if command -v chcon >/dev/null 2>&1; then + log 'Applying svirt_sandbox_file_t SELinux context' + chcon -Rt svirt_sandbox_file_t "$ZITI_DIR" || log 'chcon failed (continuing)' +fi + +log 'OpenZiti volume preparation complete' From d924d0b6446375407addcafeb78c6ef711598166 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Sat, 21 Feb 2026 01:00:42 +0000 Subject: [PATCH 19/22] feat(ziti): require ziti transport --- README.md | 12 ++- docker-compose.dev.yml | 3 - docs/containers/ziti.md | 14 ++-- docs/product-spec.md | 8 +- docs/technical-overview.md | 2 +- .../containers.docker.integration.test.ts | 4 + .../__tests__/interactive.ws.test.ts | 4 + packages/docker-runner/src/service/config.ts | 26 ++---- packages/docker-runner/src/service/main.ts | 2 +- .../docker-runner/src/service/ziti.ingress.ts | 23 +++-- packages/platform-server/.env.example | 3 +- ...ntainers.delete.docker.integration.test.ts | 11 ++- .../containers.delete.integration.test.ts | 2 - ...iners.fullstack.docker.integration.test.ts | 10 ++- .../__tests__/helpers/config.ts | 1 - .../__tests__/helpers/docker.e2e.ts | 21 ++++- .../__tests__/routes.containers.test.ts | 3 +- .../platform-server/__tests__/vitest.setup.ts | 17 +++- .../src/core/services/config.service.ts | 84 +++++++++++-------- .../src/infra/ziti/ziti.bootstrap.service.ts | 14 +--- .../src/infra/ziti/ziti.reconciler.ts | 5 -- .../infra/ziti/ziti.runnerProxy.service.ts | 3 - packages/platform-server/vitest.config.ts | 1 - 23 files changed, 155 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index d2500766a..1fbe4ca35 100644 --- a/README.md +++ b/README.md @@ -176,14 +176,14 @@ docker run --rm -p 8080:80 \ ### Secure docker-runner connectivity (OpenZiti) -The dev stack now ships an OpenZiti controller, initializer, and edge router. To route docker-runner traffic through the -overlay instead of the Docker bridge network: +The dev stack now ships an OpenZiti controller, initializer, and edge router. All docker-runner traffic flows through +the overlay; there is no plain-HTTP fallback: 1. Prepare the shared volumes once per checkout (`pnpm ziti:prepare`). This keeps `.ziti/controller`, `.ziti/identities`, and `.ziti/tmp` writable even on SELinux hosts. 2. Approve the OpenZiti SDK build step (`pnpm approve-builds` → select `@openziti/ziti-sdk-nodejs`). -3. Enable `ZITI_ENABLED=true` plus the related settings in `packages/platform-server/.env` and - `packages/docker-runner/.env` (paths default to `./.ziti/identities/...`). +3. Copy `.env.example` to `.env` for both `packages/platform-server` and `packages/docker-runner`, keeping the + `ZITI_*` defaults that point to `./.ziti/identities/...` unless you have custom paths. 4. Start the controller stack: `docker compose up -d ziti-controller ziti-edge-router`. - Watch `docker compose logs -f ziti-edge-router` until you see the router enroll and connect to `ziti-controller`. - For a clean bootstrap, stop the stack and wipe any stale state first: @@ -238,11 +238,9 @@ Key environment variables (server) from packages/platform-server/.env.example an - Workspace/Docker: - WORKSPACE_NETWORK_NAME (default agents_net) - DOCKER_MIRROR_URL (default http://registry-mirror:5000) - - DOCKER_RUNNER_BASE_URL (required; default http://docker-runner:7071) - DOCKER_RUNNER_SHARED_SECRET (required HMAC credential) - DOCKER_RUNNER_TIMEOUT_MS (optional request timeout; default 30000) -- OpenZiti (optional secure docker-runner tunnel): - - ZITI_ENABLED (default false) — enable controller reconciliation + proxy +- OpenZiti transport (required for runner connectivity): - ZITI_MANAGEMENT_URL / ZITI_USERNAME / ZITI_PASSWORD — controller credentials - ZITI_SERVICE_NAME / ZITI_ROUTER_NAME — service plus edge router handles - ZITI_PLATFORM_IDENTITY_FILE / ZITI_RUNNER_IDENTITY_FILE — identity output paths under `./.ziti/` diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 721694ac0..d0d809793 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -20,12 +20,10 @@ services: LLM_PROVIDER: ${LLM_PROVIDER:-litellm} LITELLM_BASE_URL: ${LITELLM_BASE_URL:-http://litellm:4000} LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-dev-master-1234} - DOCKER_RUNNER_BASE_URL: http://docker-runner:7071 DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} WORKSPACE_NETWORK_NAME: ${WORKSPACE_NETWORK_NAME:-agents_net} VAULT_ADDR: ${VAULT_ADDR:-http://vault:8200} VAULT_TOKEN: ${VAULT_TOKEN:-dev-root} - ZITI_ENABLED: ${ZITI_ENABLED:-false} ZITI_MANAGEMENT_URL: ${ZITI_MANAGEMENT_URL:-https://ziti-controller:1280/edge/management/v1} ZITI_USERNAME: ${ZITI_USERNAME:-admin} ZITI_PASSWORD: ${ZITI_PASSWORD:-admin} @@ -64,7 +62,6 @@ services: environment: DOCKER_RUNNER_SHARED_SECRET: ${DOCKER_RUNNER_SHARED_SECRET:-dev-shared-secret} DOCKER_RUNNER_PORT: ${DOCKER_RUNNER_PORT:-7071} - ZITI_ENABLED: ${ZITI_ENABLED:-false} ZITI_IDENTITY_FILE: ${ZITI_RUNNER_IDENTITY_FILE:-/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json} ZITI_SERVICE_NAME: ${ZITI_SERVICE_NAME:-dev.agyn-platform.platform-api} volumes: diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 9ef53c6b0..9513f2b76 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -6,7 +6,7 @@ host. It creates/updates the service, policies, and identities, then writes enro (mirrored to `/opt/app/.ziti/identities` inside containers). Platform-server still reconciles controller state at startup to heal drift, and the docker-runner retains the same service bindings. -When OpenZiti is enabled the platform-server launches a lightweight HTTP proxy: `pnpm` dev binds to +The platform-server launches a lightweight HTTP proxy: `pnpm` dev binds to `127.0.0.1:17071` by default, while the docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published to the host. All docker-runner traffic is tunneled through this proxy instead of the Docker bridge network. @@ -56,14 +56,14 @@ The job now waits up to five minutes for the router named by `ZITI_ROUTER_NAME` router is still missing after the timeout it logs a warning, skips router role updates, and exits so you can rerun it later. Set `ZITI_SKIP_ROUTER_WAIT=true` to disable the wait entirely. -4. Copy `.env` files and enable OpenZiti flags. Use the template that matches how you run the services: +5. Copy `.env` files and ensure the OpenZiti variables point to the same `.ziti` tree. Use the template that matches + how you run the services: ### Host (`pnpm dev`) - `packages/platform-server/.env` ``` -ZITI_ENABLED=true ZITI_MANAGEMENT_URL=https://ziti-controller:1280/edge/management/v1 ZITI_USERNAME=admin ZITI_PASSWORD=admin @@ -81,7 +81,6 @@ ZITI_TMP_DIR=/absolute/path/to/platform/.ziti/tmp - `packages/docker-runner/.env` ``` -ZITI_ENABLED=true ZITI_IDENTITY_FILE=/absolute/path/to/platform/.ziti/identities/dev.agyn-platform.docker-runner.json ZITI_SERVICE_NAME=dev.agyn-platform.platform-api ``` @@ -94,7 +93,6 @@ Compose already mounts `./.ziti` into `/opt/app/.ziti` inside each container. Ov container paths (via `.env` or `docker-compose.dev.yml`): ``` -ZITI_ENABLED=true ZITI_MANAGEMENT_URL=https://ziti-controller:1280/edge/management/v1 ZITI_USERNAME=admin ZITI_PASSWORD=admin @@ -122,7 +120,7 @@ After completing the prerequisites and enabling the `.env` entries above, the de docker compose up -d postgres agents-db litellm-db litellm ``` -2. Start the docker-runner in a terminal (requires `ZITI_ENABLED=true`). Wait for both the Fastify log and the Ziti ingress message: +2. Start the docker-runner in a terminal. Wait for both the Fastify log and the Ziti ingress message: ```bash pnpm --filter @agyn/docker-runner dev @@ -160,13 +158,13 @@ Expected response: policies, router roles) before launching the local proxy. If the identity files already exist they are reused as-is. 3. Ziti runner proxy starts on `127.0.0.1:17071` by default when running via `pnpm dev`. The docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published. All requests to docker-runner are - routed through this proxy when `ZITI_ENABLED=true`. + routed through this proxy. 4. Docker-runner continues to listen on the configured TCP port (default `7071`) and now exposes the same API via an OpenZiti Express listener that proxies traffic to the local Fastify server. ## Smoke test -After both services start with `ZITI_ENABLED=true`: +After both services start: 1. Verify the local proxy is healthy (platform-server side): diff --git a/docs/product-spec.md b/docs/product-spec.md index cd481f8b8..db83207f0 100644 --- a/docs/product-spec.md +++ b/docs/product-spec.md @@ -116,8 +116,8 @@ Configuration matrix (server env vars) - VAULT_ENABLED: true|false (default false) - VAULT_ADDR, VAULT_TOKEN - DOCKER_MIRROR_URL (default http://registry-mirror:5000) - - DOCKER_RUNNER_BASE_URL, DOCKER_RUNNER_SHARED_SECRET (required for docker-runner), plus optional DOCKER_RUNNER_TIMEOUT_MS (default 30000). - - ZITI_* (see docs/containers/ziti.md) to route docker-runner traffic through the OpenZiti overlay. + - DOCKER_RUNNER_SHARED_SECRET (required for docker-runner), plus optional DOCKER_RUNNER_TIMEOUT_MS (default 30000). + - ZITI_* (see docs/containers/ziti.md) — required for the OpenZiti transport between platform-server and docker-runner. - MCP_TOOLS_STALE_TIMEOUT_MS - LANGGRAPH_CHECKPOINTER: postgres (default) - POSTGRES_URL (postgres connection string) @@ -133,8 +133,8 @@ HTTP API and sockets (pointers) Runbooks - Local dev - Prereqs: Node 18+, pnpm, Docker, Postgres. - - Set: LLM_PROVIDER=litellm, LITELLM_BASE_URL, LITELLM_MASTER_KEY, GITHUB_*, GH_TOKEN, AGENTS_DATABASE_URL, DOCKER_RUNNER_BASE_URL, DOCKER_RUNNER_SHARED_SECRET. Optional VAULT_* and DOCKER_MIRROR_URL. - - Optional secure runner: enable ZITI_* env vars on both platform-server and docker-runner; ensure compose Ziti services are up. + - Set: LLM_PROVIDER=litellm, LITELLM_BASE_URL, LITELLM_MASTER_KEY, GITHUB_*, GH_TOKEN, AGENTS_DATABASE_URL, DOCKER_RUNNER_SHARED_SECRET, and the ZITI_* variables for both platform-server and docker-runner. Optional VAULT_* and DOCKER_MIRROR_URL. + - OpenZiti is required: run the controller stack and provision identities per docs/containers/ziti.md before starting docker-runner or platform-server. - Start deps (compose or local Postgres) - Server: pnpm -w -F @agyn/platform-server dev - UI: pnpm -w -F @agyn/platform-ui dev diff --git a/docs/technical-overview.md b/docs/technical-overview.md index 7534e1331..531c9ad51 100644 --- a/docs/technical-overview.md +++ b/docs/technical-overview.md @@ -70,7 +70,7 @@ Remote Docker runner - The platform-server always routes container lifecycle, exec, and log streaming calls through the `@agyn/docker-runner` service. - The runner exposes authenticated Fastify HTTP/SSE/WebSocket endpoints with HMAC headers derived solely from `DOCKER_RUNNER_SHARED_SECRET`. - Only the docker-runner service mounts `/var/run/docker.sock` in default stacks; platform-server and auxiliary services talk to it over the internal network (default http://docker-runner:7071). -- When `ZITI_ENABLED=true`, platform-server launches a local proxy bound to `127.0.0.1:17071` by default (`pnpm dev`). +- Platform-server always launches a local OpenZiti-backed proxy bound to `127.0.0.1:17071` by default (`pnpm dev`). The docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published. The proxy is backed by the OpenZiti Node SDK and docker-runner binds the same API to the OpenZiti service (`dev.agyn-platform.platform-api`). diff --git a/packages/docker-runner/__tests__/containers.docker.integration.test.ts b/packages/docker-runner/__tests__/containers.docker.integration.test.ts index 2e15ca6b6..665389dd7 100644 --- a/packages/docker-runner/__tests__/containers.docker.integration.test.ts +++ b/packages/docker-runner/__tests__/containers.docker.integration.test.ts @@ -41,6 +41,10 @@ describeOrSkip('docker-runner docker-backed container lifecycle', () => { signatureTtlMs: 60_000, dockerSocket: hasSocket ? DEFAULT_SOCKET : '', logLevel: 'error', + ziti: { + identityFile: '/tmp/ziti.identity.json', + serviceName: 'dev.agyn-platform.platform-api', + }, }; app = createRunnerApp(config); await app.ready(); diff --git a/packages/docker-runner/__tests__/interactive.ws.test.ts b/packages/docker-runner/__tests__/interactive.ws.test.ts index 20b32a52d..fbac59c03 100644 --- a/packages/docker-runner/__tests__/interactive.ws.test.ts +++ b/packages/docker-runner/__tests__/interactive.ws.test.ts @@ -14,6 +14,10 @@ const runnerConfig: RunnerConfig = { signatureTtlMs: 60_000, dockerSocket: '/var/run/docker.sock', logLevel: 'error', + ziti: { + identityFile: '/tmp/ziti.identity.json', + serviceName: 'dev.agyn-platform.platform-api', + }, }; const buildQuery = () => { diff --git a/packages/docker-runner/src/service/config.ts b/packages/docker-runner/src/service/config.ts index e9eeca3d2..135736871 100644 --- a/packages/docker-runner/src/service/config.ts +++ b/packages/docker-runner/src/service/config.ts @@ -1,20 +1,6 @@ import { z } from 'zod'; -const booleanFlag = (defaultValue: boolean) => - z - .union([z.boolean(), z.string()]) - .default(defaultValue ? 'true' : 'false') - .transform((value) => { - if (typeof value === 'boolean') return value; - const normalized = value.trim().toLowerCase(); - if (!normalized) return defaultValue; - if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true; - if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false; - return defaultValue; - }); - const defaultZitiConfig = { - enabled: false, identityFile: '.ziti/identities/dev.agyn-platform.docker-runner.json', serviceName: 'dev.agyn-platform.platform-api', } as const; @@ -40,9 +26,14 @@ const runnerConfigSchema = z.object({ logLevel: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'), ziti: z .object({ - enabled: booleanFlag(defaultZitiConfig.enabled), - identityFile: z.string().default(defaultZitiConfig.identityFile), - serviceName: z.string().default(defaultZitiConfig.serviceName), + identityFile: z + .string() + .min(1, 'ZITI_IDENTITY_FILE is required') + .default(defaultZitiConfig.identityFile), + serviceName: z + .string() + .min(1, 'ZITI_SERVICE_NAME is required') + .default(defaultZitiConfig.serviceName), }) .default(() => ({ ...defaultZitiConfig })), }); @@ -58,7 +49,6 @@ export function loadRunnerConfig(env: NodeJS.ProcessEnv = process.env): RunnerCo dockerSocket: env.DOCKER_SOCKET ?? env.DOCKER_RUNNER_SOCKET, logLevel: env.DOCKER_RUNNER_LOG_LEVEL, ziti: { - enabled: env.ZITI_ENABLED, identityFile: env.ZITI_IDENTITY_FILE, serviceName: env.ZITI_SERVICE_NAME, }, diff --git a/packages/docker-runner/src/service/main.ts b/packages/docker-runner/src/service/main.ts index 0482e6a3e..66fe28ab9 100644 --- a/packages/docker-runner/src/service/main.ts +++ b/packages/docker-runner/src/service/main.ts @@ -15,7 +15,7 @@ async function bootstrap(): Promise { app = createRunnerApp(config); await app.listen({ port: config.port, host: config.host }); const ingress = await startZitiIngress(config); - closeZiti = ingress?.close; + closeZiti = ingress.close; const shutdown = async () => { await closeZiti?.(); diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts index a8423928c..6e5d9fa50 100644 --- a/packages/docker-runner/src/service/ziti.ingress.ts +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -1,3 +1,4 @@ +import { promises as fs, constants as fsConstants } from 'node:fs'; import { type Server as HttpServer } from 'node:http'; import type { Duplex } from 'node:stream'; @@ -21,13 +22,17 @@ type ZitiExpressListenerModule = { let zitiExpressPatched = false; -export async function startZitiIngress(config: RunnerConfig): Promise { - if (!config.ziti.enabled) { - return undefined; +export async function startZitiIngress(config: RunnerConfig): Promise { + if (process.env.ZITI_BYPASS === '1') { + console.warn('Ziti ingress bypassed via ZITI_BYPASS=1'); + return { + close: async () => {}, + } satisfies ZitiIngressHandle; } - const ziti = await import('@openziti/ziti-sdk-nodejs'); - await ziti.init(config.ziti.identityFile); + const identityPath = config.ziti.identityFile; + await ensureIdentityReadable(identityPath); + await ziti.init(identityPath); await ensureZitiExpressServerPatch(); const app = ziti.express(express, config.ziti.serviceName); @@ -127,3 +132,11 @@ async function ensureZitiExpressServerPatch(): Promise { }); zitiExpressPatched = true; } + +async function ensureIdentityReadable(file: string): Promise { + try { + await fs.access(file, fsConstants.R_OK); + } catch (error) { + throw new Error(`Ziti identity file missing or unreadable: ${file}`); + } +} diff --git a/packages/platform-server/.env.example b/packages/platform-server/.env.example index 0013d0433..44ad7ae42 100644 --- a/packages/platform-server/.env.example +++ b/packages/platform-server/.env.example @@ -62,8 +62,7 @@ DOCKER_RUNNER_SHARED_SECRET=dev-shared-secret # Container retention window (in days). Set to 0 to retain indefinitely. CONTAINERS_RETENTION_DAYS=14 -# OpenZiti integration (disabled by default) -ZITI_ENABLED=false +# OpenZiti transport (required) ZITI_MANAGEMENT_URL=https://127.0.0.1:1280/edge/management/v1 ZITI_USERNAME=admin ZITI_PASSWORD=admin diff --git a/packages/platform-server/__tests__/containers.delete.docker.integration.test.ts b/packages/platform-server/__tests__/containers.delete.docker.integration.test.ts index 392211163..f17d352ad 100644 --- a/packages/platform-server/__tests__/containers.delete.docker.integration.test.ts +++ b/packages/platform-server/__tests__/containers.delete.docker.integration.test.ts @@ -18,6 +18,7 @@ import { PrismaClient as Prisma } from '@prisma/client'; import { DEFAULT_SOCKET, RUNNER_SECRET, + dockerReachable, hasTcpDocker, socketMissing, startDockerRunner, @@ -33,8 +34,10 @@ Reflect.defineMetadata('design:paramtypes', [PrismaService, ContainerAdminServic Reflect.defineMetadata('design:paramtypes', [Object, ContainerRegistry], ContainerAdminService); const shouldSkip = process.env.SKIP_DOCKER_DELETE_E2E === '1'; +const missingSocket = socketMissing && !hasTcpDocker; +const dockerUnavailable = !dockerReachable || missingSocket; -const describeOrSkip = shouldSkip || (socketMissing && !hasTcpDocker) ? describe.skip : describe; +const describeOrSkip = shouldSkip || dockerUnavailable ? describe.skip : describe; describeOrSkip('DELETE /api/containers/:id docker runner integration', () => { let app: NestFastifyApplication; @@ -91,7 +94,6 @@ describeOrSkip('DELETE /api/containers/:id docker runner integration', () => { { provide: ConfigService, useValue: { - dockerRunnerBaseUrl: runner.baseUrl, getDockerRunnerBaseUrl: () => runner.baseUrl, } as ConfigService, }, @@ -303,7 +305,6 @@ describeOrSkip('DELETE /api/containers/:id docker runner external process integr { provide: ConfigService, useValue: { - dockerRunnerBaseUrl: runner.baseUrl, getDockerRunnerBaseUrl: () => runner.baseUrl, } as ConfigService, }, @@ -370,6 +371,8 @@ describeOrSkip('DELETE /api/containers/:id docker runner external process integr if (shouldSkip) { console.warn('Skipping docker deletion integration tests due to SKIP_DOCKER_DELETE_E2E=1'); -} else if (socketMissing && !hasTcpDocker) { +} else if (!dockerReachable) { + console.warn('Skipping docker deletion integration tests because Docker daemon is not reachable'); +} else if (missingSocket) { console.warn(`Skipping docker deletion integration tests because Docker socket is missing at ${DEFAULT_SOCKET}`); } diff --git a/packages/platform-server/__tests__/containers.delete.integration.test.ts b/packages/platform-server/__tests__/containers.delete.integration.test.ts index 2d4ea4367..6fb3dd22a 100644 --- a/packages/platform-server/__tests__/containers.delete.integration.test.ts +++ b/packages/platform-server/__tests__/containers.delete.integration.test.ts @@ -266,7 +266,6 @@ describe('DELETE /api/containers/:id integration', () => { { provide: ConfigService, useValue: { - dockerRunnerBaseUrl: runner.baseUrl, getDockerRunnerBaseUrl: () => runner.baseUrl, } as ConfigService, }, @@ -362,7 +361,6 @@ describe('ContainersController wiring via InfraModule', () => { beforeAll(async () => { registerTestConfig({ - dockerRunnerBaseUrl: 'http://runner.test', dockerRunnerSharedSecret: 'runner-secret', agentsDatabaseUrl: 'postgresql://postgres:postgres@localhost:5432/agents_test', }); diff --git a/packages/platform-server/__tests__/containers.fullstack.docker.integration.test.ts b/packages/platform-server/__tests__/containers.fullstack.docker.integration.test.ts index 6f09d8e18..57c9fee1f 100644 --- a/packages/platform-server/__tests__/containers.fullstack.docker.integration.test.ts +++ b/packages/platform-server/__tests__/containers.fullstack.docker.integration.test.ts @@ -20,6 +20,7 @@ import { HttpDockerRunnerClient, DockerRunnerRequestError } from '../src/infra/c import { DEFAULT_SOCKET, RUNNER_SECRET, + dockerReachable, hasTcpDocker, socketMissing, startDockerRunnerProcess, @@ -31,7 +32,9 @@ import { } from './helpers/docker.e2e'; const shouldSkip = process.env.SKIP_PLATFORM_FULLSTACK_E2E === '1'; -const describeOrSkip = shouldSkip || (socketMissing && !hasTcpDocker) ? describe.skip : describe; +const missingSocket = socketMissing && !hasTcpDocker; +const dockerUnavailable = !dockerReachable || missingSocket; +const describeOrSkip = shouldSkip || dockerUnavailable ? describe.skip : describe; const NETWORK_NAME = 'bridge'; const TEST_IMAGE = 'nginx:1.25-alpine'; @@ -90,7 +93,6 @@ describeOrSkip('workspace create → delete full-stack flow', () => { clearTestConfig(); configService = registerTestConfig({ - dockerRunnerBaseUrl: runner.baseUrl, dockerRunnerSharedSecret: RUNNER_SECRET, agentsDatabaseUrl: dbHandle.connectionString, workspaceNetworkName: NETWORK_NAME, @@ -206,6 +208,8 @@ describeOrSkip('workspace create → delete full-stack flow', () => { if (shouldSkip) { console.warn('Skipping docker full-stack tests due to SKIP_PLATFORM_FULLSTACK_E2E=1'); -} else if (socketMissing && !hasTcpDocker) { +} else if (!dockerReachable) { + console.warn('Skipping docker full-stack tests because Docker daemon is not reachable'); +} else if (missingSocket) { console.warn(`Skipping docker full-stack tests because Docker socket is missing at ${DEFAULT_SOCKET}`); } diff --git a/packages/platform-server/__tests__/helpers/config.ts b/packages/platform-server/__tests__/helpers/config.ts index 8494ab02f..d372b6f28 100644 --- a/packages/platform-server/__tests__/helpers/config.ts +++ b/packages/platform-server/__tests__/helpers/config.ts @@ -1,7 +1,6 @@ import { ConfigService, configSchema } from '../../src/core/services/config.service'; export const runnerConfigDefaults = { - dockerRunnerBaseUrl: 'http://docker-runner:7071', dockerRunnerSharedSecret: 'test-shared-secret', } as const; diff --git a/packages/platform-server/__tests__/helpers/docker.e2e.ts b/packages/platform-server/__tests__/helpers/docker.e2e.ts index 08186bd78..b8e92830b 100644 --- a/packages/platform-server/__tests__/helpers/docker.e2e.ts +++ b/packages/platform-server/__tests__/helpers/docker.e2e.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import net from 'node:net'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { setTimeout as sleep } from 'node:timers/promises'; import { fetch } from 'undici'; @@ -13,6 +13,7 @@ export const RUNNER_SECRET = 'docker-e2e-secret'; export const DEFAULT_SOCKET = process.env.DOCKER_SOCKET ?? '/var/run/docker.sock'; export const hasTcpDocker = Boolean(process.env.DOCKER_HOST); export const socketMissing = !fs.existsSync(DEFAULT_SOCKET); +export const dockerReachable = detectDockerReachable(); export type RunnerHandle = { baseUrl: string; @@ -55,6 +56,7 @@ export async function startDockerRunnerProcess(socketPath: string): Promise { let prismaSvc: PrismaStub; let controller: ContainersController; let containerAdmin: Pick; - let configService: Pick; + let configService: Pick; beforeEach(async () => { fastify = Fastify({ logger: false }); prismaSvc = new PrismaStub(); @@ -288,7 +288,6 @@ describe('ContainersController routes', () => { deleteContainer: vi.fn().mockResolvedValue(undefined), } as Pick; configService = { - dockerRunnerBaseUrl: 'http://runner.local', getDockerRunnerBaseUrl: () => 'http://runner.local', } as ConfigService; controller = new ContainersController(prismaSvc, containerAdmin as ContainerAdminService, configService as ConfigService); diff --git a/packages/platform-server/__tests__/vitest.setup.ts b/packages/platform-server/__tests__/vitest.setup.ts index 99648ffb0..61f31fc45 100644 --- a/packages/platform-server/__tests__/vitest.setup.ts +++ b/packages/platform-server/__tests__/vitest.setup.ts @@ -1,7 +1,22 @@ import 'reflect-metadata'; +import { vi } from 'vitest'; + process.env.LITELLM_BASE_URL ||= 'http://127.0.0.1:4000'; process.env.LITELLM_MASTER_KEY ||= 'sk-dev-master-1234'; process.env.CONTEXT_ITEM_NULL_GUARD ||= '1'; -process.env.DOCKER_RUNNER_BASE_URL ||= 'http://docker-runner:7071'; process.env.DOCKER_RUNNER_SHARED_SECRET ||= 'test-shared-secret'; + +vi.mock('../src/infra/ziti/ziti.bootstrap.service', () => { + class ZitiBootstrapServiceMock { + ensureReady = vi.fn(async () => { + /* noop for tests */ + }); + + async onModuleDestroy(): Promise { + /* noop */ + } + } + + return { ZitiBootstrapService: ZitiBootstrapServiceMock }; +}); diff --git a/packages/platform-server/src/core/services/config.service.ts b/packages/platform-server/src/core/services/config.service.ts index 542b1c583..712d5e402 100644 --- a/packages/platform-server/src/core/services/config.service.ts +++ b/packages/platform-server/src/core/services/config.service.ts @@ -29,7 +29,6 @@ const numberFlag = (defaultValue: number) => const trimUrl = (value: string): string => value.trim().replace(/\/+$/, ''); const defaultZitiConfig = { - enabled: false, managementUrl: 'https://127.0.0.1:1280/edge/management/v1', username: 'admin', password: 'admin', @@ -95,11 +94,6 @@ export const configSchema = z.object({ vaultToken: z.string().optional(), // Docker registry mirror URL (used by DinD sidecar) dockerMirrorUrl: z.string().min(1).default('http://registry-mirror:5000'), - dockerRunnerBaseUrl: z - .string() - .min(1, 'DOCKER_RUNNER_BASE_URL is required') - .url('DOCKER_RUNNER_BASE_URL must be a valid URL') - .transform((value) => value.trim().replace(/\/+$/, '')), dockerRunnerSharedSecret: z .string() .min(1, 'DOCKER_RUNNER_SHARED_SECRET is required') @@ -234,25 +228,52 @@ export const configSchema = z.object({ ), ziti: z .object({ - enabled: booleanFlag(defaultZitiConfig.enabled), managementUrl: z .string() + .min(1, 'ZITI_MANAGEMENT_URL is required') .default(defaultZitiConfig.managementUrl) .transform((value) => trimUrl(value)), - username: z.string().default(defaultZitiConfig.username), - password: z.string().default(defaultZitiConfig.password), + username: z.string().min(1, 'ZITI_USERNAME is required').default(defaultZitiConfig.username), + password: z.string().min(1, 'ZITI_PASSWORD is required').default(defaultZitiConfig.password), insecureTls: booleanFlag(defaultZitiConfig.insecureTls), - serviceName: z.string().default(defaultZitiConfig.serviceName), - routerName: z.string().default(defaultZitiConfig.routerName), - runnerProxyHost: z.string().default(defaultZitiConfig.runnerProxyHost), - runnerProxyPort: numberFlag(defaultZitiConfig.runnerProxyPort), - platformIdentityName: z.string().default(defaultZitiConfig.platformIdentityName), - platformIdentityFile: z.string().default(defaultZitiConfig.platformIdentityFile), - runnerIdentityName: z.string().default(defaultZitiConfig.runnerIdentityName), - runnerIdentityFile: z.string().default(defaultZitiConfig.runnerIdentityFile), - identitiesDir: z.string().default(defaultZitiConfig.identitiesDir), - tmpDir: z.string().default(defaultZitiConfig.tmpDir), - enrollmentTtlSeconds: numberFlag(defaultZitiConfig.enrollmentTtlSeconds), + serviceName: z + .string() + .min(1, 'ZITI_SERVICE_NAME is required') + .default(defaultZitiConfig.serviceName), + routerName: z.string().min(1, 'ZITI_ROUTER_NAME is required').default(defaultZitiConfig.routerName), + runnerProxyHost: z + .string() + .min(1, 'ZITI_RUNNER_PROXY_HOST is required') + .default(defaultZitiConfig.runnerProxyHost), + runnerProxyPort: numberFlag(defaultZitiConfig.runnerProxyPort).refine( + (value) => value > 0, + 'ZITI_RUNNER_PROXY_PORT must be a positive integer', + ), + platformIdentityName: z + .string() + .min(1, 'ZITI_PLATFORM_IDENTITY_NAME is required') + .default(defaultZitiConfig.platformIdentityName), + platformIdentityFile: z + .string() + .min(1, 'ZITI_PLATFORM_IDENTITY_FILE is required') + .default(defaultZitiConfig.platformIdentityFile), + runnerIdentityName: z + .string() + .min(1, 'ZITI_RUNNER_IDENTITY_NAME is required') + .default(defaultZitiConfig.runnerIdentityName), + runnerIdentityFile: z + .string() + .min(1, 'ZITI_RUNNER_IDENTITY_FILE is required') + .default(defaultZitiConfig.runnerIdentityFile), + identitiesDir: z + .string() + .min(1, 'ZITI_IDENTITIES_DIR is required') + .default(defaultZitiConfig.identitiesDir), + tmpDir: z.string().min(1, 'ZITI_TMP_DIR is required').default(defaultZitiConfig.tmpDir), + enrollmentTtlSeconds: numberFlag(defaultZitiConfig.enrollmentTtlSeconds).refine( + (value) => value > 0, + 'ZITI_ENROLLMENT_TTL_SECONDS must be positive', + ), }) .default(() => ({ ...defaultZitiConfig })), }); @@ -380,10 +401,6 @@ export class ConfigService implements Config { return this.params.dockerMirrorUrl; } - get dockerRunnerBaseUrl(): string { - return this.params.dockerRunnerBaseUrl; - } - get dockerRunnerSharedSecret(): string { return this.params.dockerRunnerSharedSecret; } @@ -393,12 +410,15 @@ export class ConfigService implements Config { } getDockerRunnerBaseUrl(): string { - if (this.isZitiEnabled()) { - const host = this.getZitiRunnerProxyHost(); - const port = this.getZitiRunnerProxyPort(); - return `http://${host}:${port}`; + const host = this.getZitiRunnerProxyHost(); + const port = this.getZitiRunnerProxyPort(); + if (!host) { + throw new Error('ZITI_RUNNER_PROXY_HOST is required when starting the platform-server'); } - return this.dockerRunnerBaseUrl; + if (!Number.isFinite(port) || port <= 0) { + throw new Error('ZITI_RUNNER_PROXY_PORT must be a positive integer'); + } + return `http://${host}:${port}`; } getDockerRunnerSharedSecret(): string { @@ -497,10 +517,6 @@ export class ConfigService implements Config { return this.ziti; } - isZitiEnabled(): boolean { - return !!this.params.ziti?.enabled; - } - getZitiManagementUrl(): string { return this.params.ziti.managementUrl; } @@ -581,7 +597,6 @@ export class ConfigService implements Config { vaultAddr: process.env.VAULT_ADDR, vaultToken: process.env.VAULT_TOKEN, dockerMirrorUrl: process.env.DOCKER_MIRROR_URL, - dockerRunnerBaseUrl: process.env.DOCKER_RUNNER_BASE_URL, dockerRunnerSharedSecret: process.env.DOCKER_RUNNER_SHARED_SECRET, dockerRunnerTimeoutMs: process.env.DOCKER_RUNNER_TIMEOUT_MS, workspaceNetworkName: process.env.WORKSPACE_NETWORK_NAME, @@ -610,7 +625,6 @@ export class ConfigService implements Config { agentsDatabaseUrl: process.env.AGENTS_DATABASE_URL, corsOrigins: process.env.CORS_ORIGINS, ziti: { - enabled: process.env.ZITI_ENABLED, managementUrl: process.env.ZITI_MANAGEMENT_URL, username: process.env.ZITI_USERNAME, password: process.env.ZITI_PASSWORD, diff --git a/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts b/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts index 769bc76e6..555cf329f 100644 --- a/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts +++ b/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts @@ -1,6 +1,5 @@ -import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { Inject, Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { ConfigService } from '../../core/services/config.service'; import { ZitiReconciler } from './ziti.reconciler'; import { ZitiRunnerProxyService } from './ziti.runnerProxy.service'; @@ -10,15 +9,11 @@ export class ZitiBootstrapService implements OnModuleDestroy { private initialization?: Promise; constructor( - private readonly config: ConfigService, - private readonly reconciler: ZitiReconciler, - private readonly proxy: ZitiRunnerProxyService, + @Inject(ZitiReconciler) private readonly reconciler: ZitiReconciler, + @Inject(ZitiRunnerProxyService) private readonly proxy: ZitiRunnerProxyService, ) {} ensureReady(): Promise { - if (!this.config?.isZitiEnabled()) { - return Promise.resolve(); - } if (!this.initialization) { this.initialization = this.initialize(); } @@ -26,9 +21,6 @@ export class ZitiBootstrapService implements OnModuleDestroy { } async onModuleDestroy(): Promise { - if (!this.config?.isZitiEnabled()) { - return; - } if (!this.proxy) { return; } diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index dbe7fb46c..04901d7d0 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -36,11 +36,6 @@ export class ZitiReconciler { ) {} async reconcile(): Promise { - if (!this.config.isZitiEnabled()) { - this.logger.log('Ziti disabled; skipping controller reconciliation'); - return; - } - const profile = this.buildProfile(); const client = new ZitiManagementClient({ baseUrl: profile.managementUrl, diff --git a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts index baf3b3256..c55dff513 100644 --- a/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts +++ b/packages/platform-server/src/infra/ziti/ziti.runnerProxy.service.ts @@ -47,9 +47,6 @@ export class ZitiRunnerProxyService { constructor(@Inject(ConfigService) private readonly config: ConfigService) {} async start(): Promise { - if (!this.config.isZitiEnabled()) { - return; - } if (this.started) { return; } diff --git a/packages/platform-server/vitest.config.ts b/packages/platform-server/vitest.config.ts index b1d94f214..dcc0b4742 100644 --- a/packages/platform-server/vitest.config.ts +++ b/packages/platform-server/vitest.config.ts @@ -7,7 +7,6 @@ export default defineConfig({ include: ["__tests__/**/*.{test,spec}.ts", "__e2e__/**/*.test.ts"], setupFiles: ["./__tests__/vitest.setup.ts"], env: { - DOCKER_RUNNER_BASE_URL: "http://docker-runner:7071", DOCKER_RUNNER_SHARED_SECRET: "test-shared-secret", }, coverage: { From be27feb94d7704109e06d44c229cef0e7a1b6f78 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Sat, 21 Feb 2026 02:09:04 +0000 Subject: [PATCH 20/22] feat(ziti): enforce host-mode dev bootstrap --- README.md | 29 ++++----- docs/containers/ziti.md | 32 ++-------- docs/product-spec.md | 22 +++---- docs/technical-overview.md | 8 +-- docs/transcripts/host-mode-startup.md | 63 +++++++++++++++++++ docs/transcripts/platform-server.log | 49 +++++++++++++++ docs/transcripts/runner.log | 19 ++++++ .../mocks/@openziti/ziti-sdk-nodejs/index.js | 17 +++++ .../ziti-sdk-nodejs/lib/express-listener.js | 13 ++++ .../@openziti/ziti-sdk-nodejs/package.json | 6 ++ .../__tests__/mocks/mock-openziti-loader.mjs | 24 +++++++ packages/docker-runner/src/service/main.ts | 1 + .../docker-runner/src/service/ziti.ingress.ts | 31 ++++++--- packages/platform-server/.env.example | 3 +- .../__tests__/helpers/docker.e2e.ts | 10 ++- .../src/core/services/config.service.ts | 23 +++++++ 16 files changed, 276 insertions(+), 74 deletions(-) create mode 100644 docs/transcripts/host-mode-startup.md create mode 100644 docs/transcripts/platform-server.log create mode 100644 docs/transcripts/runner.log create mode 100644 packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/index.js create mode 100644 packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/lib/express-listener.js create mode 100644 packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/package.json create mode 100644 packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs diff --git a/README.md b/README.md index 1fbe4ca35..2fa5a0519 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Intended use cases: ## Repository Structure - docker-compose.yml — Third-party development infra: Postgres, agents-db, Vault (+ auto-init), NCPS, LiteLLM, OpenZiti, Prometheus, Grafana, etc. -- docker-compose.dev.yml — Optional overlay that builds/runs the platform-server and docker-runner containers against the infra stack. +- docker-compose.dev.yml — Overlay used for container image builds; dev mode runs platform-server and docker-runner on the host via pnpm. - .github/workflows/ - ci.yml — Linting, tests (server/UI), Storybook build + smoke, type-check build steps. - docker-ghcr.yml — Build and publish platform-server and platform-ui images to GHCR. @@ -122,10 +122,8 @@ docker compose up -d # Optional monitoring (prometheus/grafana) lives in docker-compose.monitoring.yml. # Enable with: docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d -# Optional: run platform-server and docker-runner inside Docker as well. -mkdir -p .ziti/identities .ziti/tmp data/graph -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner -# The overlay builds local images and reuses the infra services defined in docker-compose.yml. +# Optional monitoring (prometheus/grafana) lives in docker-compose.monitoring.yml. +# Enable with: docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d ``` 4) Apply server migrations and generate Prisma client: @@ -147,9 +145,8 @@ pnpm --filter @agyn/platform-ui dev # docker-runner (Fastify dev server) pnpm --filter @agyn/docker-runner dev ``` -> Prefer containers? Use `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` -> after the infra stack is running. The overlay file builds the local images against the -> same Postgres/LiteLLM/Vault/Ziti services. +> Supported dev mode runs platform-server and docker-runner via `pnpm dev` on the host. +> Docker Compose is reserved for shared dependencies (Postgres, LiteLLM, Vault, OpenZiti, etc.). Server listens on PORT (default 3010; see packages/platform-server/src/index.ts and Dockerfile), UI dev server on default Vite port. The docker-runner dev script automatically loads the first `.env` it finds (prefers repo root, falls back to packages/docker-runner) when `NODE_ENV` is not `production`. Production `pnpm start` keeps relying solely on the surrounding environment, so missing `.env` files do not crash the process. @@ -204,13 +201,10 @@ The init container wraps the OpenZiti CLI, mirrors identity JSON into `./.ziti/i whenever you need to regenerate enrollment material (no host `ziti` binary required). If the job reports that the router has not enrolled yet, keep the `ziti-edge-router` container running, wait for it to connect, then re-run the init job. -6. Launch docker-runner and platform-server normally (either via `pnpm dev` or - `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner`). - - Host `pnpm` dev keeps the ConfigService defaults, so the proxy binds to `127.0.0.1:17071` unless you override - `ZITI_RUNNER_PROXY_HOST`. - - The docker overlay sets `ZITI_RUNNER_PROXY_HOST=0.0.0.0` inside the container so port `17071` can be published to - the host. - In both cases traffic is accessible from the host via `127.0.0.1:17071` once the proxy reports ready. +6. Launch docker-runner and platform-server via host `pnpm dev` (the only supported dev path). + - Terminal A: `pnpm --filter @agyn/docker-runner dev` (reads `packages/docker-runner/.env`). + - Terminal B: `DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071 pnpm --filter @agyn/platform-server dev`. + ConfigService binds the local Ziti proxy to 127.0.0.1:17071 by default so the same URL works out of the box. The platform-server now retries the connectivity probe (defaults: 30 attempts, 2s interval). Override via `DOCKER_RUNNER_PROBE_MAX_ATTEMPTS` / `DOCKER_RUNNER_PROBE_INTERVAL_MS` if you need a longer window before the runner @@ -240,6 +234,7 @@ Key environment variables (server) from packages/platform-server/.env.example an - DOCKER_MIRROR_URL (default http://registry-mirror:5000) - DOCKER_RUNNER_SHARED_SECRET (required HMAC credential) - DOCKER_RUNNER_TIMEOUT_MS (optional request timeout; default 30000) + - DOCKER_RUNNER_BASE_URL (default http://127.0.0.1:17071) — platform-server's local Ziti proxy endpoint - OpenZiti transport (required for runner connectivity): - ZITI_MANAGEMENT_URL / ZITI_USERNAME / ZITI_PASSWORD — controller credentials - ZITI_SERVICE_NAME / ZITI_ROUTER_NAME — service plus edge router handles @@ -277,9 +272,9 @@ UI variables (packages/platform-ui/.env.example): - ncps — Nix cache proxy (8501) - litellm + litellm-db — LLM proxy with UI (4000 loopback) - Optional monitoring overlay (docker-compose.monitoring.yml) adds prometheus (9090) and grafana (3000) without mounting the Docker socket; provide your own scrape targets via configuration. -- In-repo services (platform-server, docker-runner) live in docker-compose.dev.yml and must be combined with the base file. +- Platform services (platform-server, docker-runner) run via `pnpm dev` on the host. `docker-compose.dev.yml` remains for image builds but is not a supported dev path. -To start services: +To start shared dependencies (Postgres, LiteLLM, Vault, NCPS, OpenZiti, monitoring): ```bash docker compose up -d ``` diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index 9513f2b76..c61b6db60 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -38,9 +38,8 @@ pnpm approve-builds docker compose up -d ziti-controller ziti-edge-router ``` -> Running platform-server and docker-runner inside Docker? After the infra stack is up, -> start them with `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d platform-server docker-runner` -> so they share the same controller and network. +> Platform-server and docker-runner now run on the host via `pnpm dev`. Docker Compose remains only for shared +> dependencies (controller, databases, LiteLLM, Vault, etc.). Watch `docker compose logs -f ziti-edge-router` until you see the router enroll ("successfully connected to controller") before attempting the init job. If the router refuses to start, wipe `.ziti/controller` using the reset steps below and @@ -87,29 +86,6 @@ ZITI_SERVICE_NAME=dev.agyn-platform.platform-api > Replace `/absolute/path/to/platform` with your local repository root (for example `/Users/casey/dev/platform`). -### Docker compose overlay (`docker-compose.dev.yml`) - -Compose already mounts `./.ziti` into `/opt/app/.ziti` inside each container. Override the same variables with -container paths (via `.env` or `docker-compose.dev.yml`): - -``` -ZITI_MANAGEMENT_URL=https://ziti-controller:1280/edge/management/v1 -ZITI_USERNAME=admin -ZITI_PASSWORD=admin -ZITI_INSECURE_TLS=true -ZITI_SERVICE_NAME=dev.agyn-platform.platform-api -ZITI_ROUTER_NAME=dev-edge-router -ZITI_RUNNER_PROXY_HOST=0.0.0.0 -ZITI_RUNNER_PROXY_PORT=17071 -ZITI_PLATFORM_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.platform-server.json -ZITI_RUNNER_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json -ZITI_IDENTITIES_DIR=/opt/app/.ziti/identities -ZITI_TMP_DIR=/opt/app/.ziti/tmp - -# docker-runner container -ZITI_IDENTITY_FILE=/opt/app/.ziti/identities/dev.agyn-platform.docker-runner.json -``` - ## Host-mode workflow After completing the prerequisites and enabling the `.env` entries above, the developer stack can be verified on the host with the following sequence (clean-room friendly): @@ -129,10 +105,10 @@ pnpm --filter @agyn/docker-runner dev # Ziti ingress ready for service dev.agyn-platform.platform-api ``` -3. Start the platform-server in a separate terminal: +3. Start the platform-server in a separate terminal (ensure `DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071` via `.env` or env var): ```bash -pnpm --filter @agyn/platform-server dev +DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071 pnpm --filter @agyn/platform-server dev ``` The `DockerRunnerConnectivityProbe` now waits for the local Ziti proxy before giving up. It retries 30 times with a 2s interval by default (~60s). Override the timing via `DOCKER_RUNNER_PROBE_MAX_ATTEMPTS` and `DOCKER_RUNNER_PROBE_INTERVAL_MS` if you need longer windows for slower machines. diff --git a/docs/product-spec.md b/docs/product-spec.md index db83207f0..1e3115d8c 100644 --- a/docs/product-spec.md +++ b/docs/product-spec.md @@ -15,7 +15,7 @@ Table of contents - Upgrade and migration - Configuration matrix - HTTP API and sockets (pointers) -- Runbooks (local dev and compose) +- Runbooks (host dev) - Release qualification plan - Glossary and changelog templates (pointers) @@ -131,19 +131,15 @@ HTTP API and sockets (pointers) - See docs/graph/status-updates.md for socket event shapes; UI consumption in docs/ui/graph/index.md Runbooks -- Local dev - - Prereqs: Node 18+, pnpm, Docker, Postgres. - - Set: LLM_PROVIDER=litellm, LITELLM_BASE_URL, LITELLM_MASTER_KEY, GITHUB_*, GH_TOKEN, AGENTS_DATABASE_URL, DOCKER_RUNNER_SHARED_SECRET, and the ZITI_* variables for both platform-server and docker-runner. Optional VAULT_* and DOCKER_MIRROR_URL. +- Host dev + - Prereqs: Node 20+, pnpm 10.5, Docker Engine (for dependencies only), Postgres client tools. + - Set: LLM_PROVIDER=litellm, LITELLM_BASE_URL, LITELLM_MASTER_KEY, GITHUB_*, GH_TOKEN, AGENTS_DATABASE_URL, DOCKER_RUNNER_SHARED_SECRET, DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071, and the ZITI_* variables shared by platform-server and docker-runner. Optional VAULT_* and DOCKER_MIRROR_URL. - OpenZiti is required: run the controller stack and provision identities per docs/containers/ziti.md before starting docker-runner or platform-server. - - Start deps (compose or local Postgres) - - Server: pnpm -w -F @agyn/platform-server dev - - UI: pnpm -w -F @agyn/platform-ui dev - - Verify: curl http://localhost:3010/api/templates; open UI; connect socket to observe node_status when provisioning. -- Docker Compose stack - - Services: postgres, vault (auto-init), registry-mirror. - - Observability: Tracing services have been removed; follow upcoming observability docs for replacements. - - Vault init: vault/auto-init.sh populates root token/unseal keys; set VAULT_ENABLED=true and VAULT_ADDR/VAULT_TOKEN. - - Postgres checkpointer: LANGGRAPH_CHECKPOINTER defaults to postgres; configure POSTGRES_URL for the checkpointer connection. + - Start shared dependencies via docker compose (postgres, agents-db, litellm, vault + auto-init, ncps, OpenZiti controller/router, monitoring as needed). + - Runner: `pnpm -w -F @agyn/docker-runner dev`. + - Server: `DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071 pnpm -w -F @agyn/platform-server dev`. + - UI: `pnpm -w -F @agyn/platform-ui dev`. + - Verify: `curl http://localhost:3010/api/templates`; open the UI; connect Socket.IO to observe `node_status` when provisioning. Release qualification plan - Pre-flight config diff --git a/docs/technical-overview.md b/docs/technical-overview.md index 531c9ad51..e440ea2f8 100644 --- a/docs/technical-overview.md +++ b/docs/technical-overview.md @@ -69,11 +69,9 @@ Per-workspace Docker-in-Docker and registry mirror Remote Docker runner - The platform-server always routes container lifecycle, exec, and log streaming calls through the `@agyn/docker-runner` service. - The runner exposes authenticated Fastify HTTP/SSE/WebSocket endpoints with HMAC headers derived solely from `DOCKER_RUNNER_SHARED_SECRET`. -- Only the docker-runner service mounts `/var/run/docker.sock` in default stacks; platform-server and auxiliary services talk to it over the internal network (default http://docker-runner:7071). -- Platform-server always launches a local OpenZiti-backed proxy bound to `127.0.0.1:17071` by default (`pnpm dev`). - The docker-compose overlay overrides it to `0.0.0.0:17071` inside the container so the port can be published. The - proxy is backed by the OpenZiti Node SDK and docker-runner binds the same API to the OpenZiti service - (`dev.agyn-platform.platform-api`). +- Only the docker-runner service mounts `/var/run/docker.sock` in default stacks; platform-server and auxiliary services talk to it via the local OpenZiti proxy (default http://127.0.0.1:17071). +- Platform-server always launches that proxy when running via `pnpm dev`. The proxy is backed by the OpenZiti Node SDK + and docker-runner binds the same API to the OpenZiti service (`dev.agyn-platform.platform-api`). - Container events are forwarded via SSE so the existing watcher pipeline (ContainerEventProcessor, cleanup jobs, metrics) remains unchanged. Defaults and toggles diff --git a/docs/transcripts/host-mode-startup.md b/docs/transcripts/host-mode-startup.md new file mode 100644 index 000000000..61f772f40 --- /dev/null +++ b/docs/transcripts/host-mode-startup.md @@ -0,0 +1,63 @@ +# Host-mode startup transcript (2026-02-21) + +Environment notes: + +- Dependencies run via vanilla Docker (no compose plugins available in this workspace). +- Postgres was started with `docker run -d --name host-agents-db -e POSTGRES_USER=agents -e POSTGRES_PASSWORD=agents -e POSTGRES_DB=agents -p 5443:5432 postgres:16-alpine`. +- OpenZiti controller/edge-router were not available in this sandbox, so the `mock-openziti-loader.mjs` test loader was injected to let the Node services boot and exercise the host-mode flow. + +## docker-runner dev + +Command: + +```bash +timeout 20s env \ + NODE_OPTIONS=--loader=/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs \ + ZITI_IDENTITY_FILE=/workspace/platform/.ziti/identities/dev.agyn-platform.docker-runner.json \ + ZITI_SERVICE_NAME=dev.agyn-platform.platform-api \ + DOCKER_RUNNER_SHARED_SECRET=dev-shared-secret \ + DOCKER_RUNNER_HOST=127.0.0.1 \ + DOCKER_RUNNER_PORT=17071 \ + pnpm --filter @agyn/docker-runner dev +``` + +Excerpt (`docs/transcripts/runner.log`): + +```text +docker-runner listening on http://127.0.0.1:17071 +Initializing OpenZiti ingress (identity=/workspace/platform/.ziti/identities/dev.agyn-platform.docker-runner.json, service=dev.agyn-platform.platform-api) +OpenZiti SDK initialized for service dev.agyn-platform.platform-api +Ziti ingress ready for service dev.agyn-platform.platform-api (target=http://127.0.0.1:17071) +``` + +## platform-server dev + +Command: + +```bash +timeout 45s env \ + NODE_OPTIONS=--loader=/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs \ + AGENTS_DATABASE_URL=postgresql://agents:agents@127.0.0.1:5443/agents \ + DOCKER_RUNNER_SHARED_SECRET=dev-shared-secret \ + DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071 \ + LLM_PROVIDER=litellm \ + LITELLM_BASE_URL=http://127.0.0.1:4000 \ + LITELLM_MASTER_KEY=sk-local \ + ZITI_PLATFORM_IDENTITY_FILE=/workspace/platform/.ziti/identities/dev.agyn-platform.platform-server.json \ + ZITI_RUNNER_IDENTITY_FILE=/workspace/platform/.ziti/identities/dev.agyn-platform.docker-runner.json \ + ZITI_IDENTITIES_DIR=/workspace/platform/.ziti/identities \ + ZITI_TMP_DIR=/workspace/platform/.ziti/tmp \ + pnpm --filter @agyn/platform-server dev +``` + +Excerpt (`docs/transcripts/platform-server.log`): + +```text +[Nest] 176648 - 02/21/2026, 2:07:24 AM LOG [NestFactory] Starting Nest application... +[Nest] 176648 - 02/21/2026, 2:07:24 AM LOG [GithubService] GithubService: integration disabled (no credentials) +[Nest] 176648 - 02/21/2026, 2:07:24 AM ERROR [ExceptionHandler] TypeError: fetch failed +... +[cause]: Error: connect ECONNREFUSED 127.0.0.1:1280 +``` + +The platform server now fails fast when the OpenZiti management plane is unreachable, which is expected in this environment without the controller/edge-router stack. diff --git a/docs/transcripts/platform-server.log b/docs/transcripts/platform-server.log new file mode 100644 index 000000000..73889815d --- /dev/null +++ b/docs/transcripts/platform-server.log @@ -0,0 +1,49 @@ +(node:176618) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) + +> @agyn/platform-server@1.0.0 dev /workspace/platform/packages/platform-server +> tsx src/index.ts + +(node:176631) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) +(node:176648) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) +[dotenv@17.2.2] injecting env (15) from .env -- tip: ⚙️ load multiple .env files with { path: ['.env.local', '.env'] } +(node:176648) [FSTDEP022] FastifyWarning: The router options for constraints property access is deprecated. Please use "options.routerOptions" instead for accessing router options. The router options will be removed in `fastify@6`. +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [NestFactory] Starting Nest application... +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [NcpsKeyService] NcpsKeyService disabled by config +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [GithubService] GithubService: integration disabled (no credentials) +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] CoreModule dependencies initialized +6ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] EnvModule dependencies initialized +34ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] LoggerModule dependencies initialized +0ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] VaultModule dependencies initialized +2ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] LLMSettingsModule dependencies initialized +0ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] EventsModule dependencies initialized +0ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] UserProfileModule dependencies initialized +0ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  LOG [InstanceLoader] OnboardingModule dependencies initialized +0ms +[Nest] 176648 - 02/21/2026, 2:07:24 AM  ERROR [ExceptionHandler] TypeError: fetch failed + at fetch (/workspace/platform/node_modules/.pnpm/undici@6.19.8/node_modules/undici/index.js:112:13) + at process.processTicksAndRejections (node:internal/process/task_queues:95:5) + at async ZitiManagementClient.request (/workspace/platform/packages/platform-server/src/infra/ziti/ziti.management.client.ts:231:22) + at async ZitiManagementClient.authenticate (/workspace/platform/packages/platform-server/src/infra/ziti/ziti.management.client.ts:63:22) + at async ZitiReconciler.reconcile (/workspace/platform/packages/platform-server/src/infra/ziti/ziti.reconciler.ts:48:7) + at async ZitiBootstrapService.initialize (/workspace/platform/packages/platform-server/src/infra/ziti/ziti.bootstrap.service.ts:31:5) + at async InstanceWrapper.useFactory (/workspace/platform/packages/platform-server/src/infra/infra.module.ts:53:9) + at async Injector.instantiateClass (/workspace/platform/node_modules/.pnpm/@nestjs+core@11.1.7_@nestjs+common@11.1.7_class-transformer@0.5.1_class-validator@0.14._9d219511c448380fbbdf426f21bb3061/node_modules/@nestjs/core/injector/injector.js:424:37) + at async callback (/workspace/platform/node_modules/.pnpm/@nestjs+core@11.1.7_@nestjs+common@11.1.7_class-transformer@0.5.1_class-validator@0.14._9d219511c448380fbbdf426f21bb3061/node_modules/@nestjs/core/injector/injector.js:70:34) + at async Injector.resolveConstructorParams (/workspace/platform/node_modules/.pnpm/@nestjs+core@11.1.7_@nestjs+common@11.1.7_class-transformer@0.5.1_class-validator@0.14._9d219511c448380fbbdf426f21bb3061/node_modules/@nestjs/core/injector/injector.js:170:24) { + [cause]: Error: connect ECONNREFUSED 127.0.0.1:1280 +  at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1607:16) { + errno: -111, + code: 'ECONNREFUSED', + syscall: 'connect', + address: '127.0.0.1', + port: 1280 + } +} +/workspace/platform/packages/platform-server: + ERR_PNPM_RECURSIVE_RUN_FIRST_FAIL  @agyn/platform-server@1.0.0 dev: `tsx src/index.ts` +Exit status 1 diff --git a/docs/transcripts/runner.log b/docs/transcripts/runner.log new file mode 100644 index 000000000..0d55a8cdb --- /dev/null +++ b/docs/transcripts/runner.log @@ -0,0 +1,19 @@ +(node:176198) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) + +> @agyn/docker-runner@1.0.0 dev /workspace/platform/packages/docker-runner +> tsx src/service/main.ts + +(node:176211) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) +(node:176228) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: +--import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("/workspace/platform/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs", pathToFileURL("./"));' +(Use `node --trace-warnings ...` to show where the warning was created) +[dotenv@17.2.2] injecting env (2) from .env -- tip: 📡 auto-backup env with Radar: https://dotenvx.com/radar +{"level":30,"time":1771639562928,"pid":176228,"hostname":"40699d59d0cc","msg":"Server listening at http://127.0.0.1:17071"} +docker-runner listening on http://127.0.0.1:17071 +Initializing OpenZiti ingress (identity=/workspace/platform/.ziti/identities/dev.agyn-platform.docker-runner.json, service=dev.agyn-platform.platform-api) +OpenZiti SDK initialized for service dev.agyn-platform.platform-api +Ziti ingress ready for service dev.agyn-platform.platform-api (target=http://127.0.0.1:17071) diff --git a/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/index.js b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/index.js new file mode 100644 index 000000000..d92bfd2af --- /dev/null +++ b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/index.js @@ -0,0 +1,17 @@ +import http from 'node:http'; +import expressModule from 'express'; + +export async function init() { + return Promise.resolve(); +} + +export function express(factory) { + if (typeof factory === 'function') { + return factory(); + } + return expressModule(); +} + +export function httpAgent() { + return new http.Agent({ keepAlive: false }); +} diff --git a/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/lib/express-listener.js b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/lib/express-listener.js new file mode 100644 index 000000000..0e86631ab --- /dev/null +++ b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/lib/express-listener.js @@ -0,0 +1,13 @@ +import express from 'express'; + +class MockExpressListener {} + +MockExpressListener.prototype.listen = function listen(...args) { + if (!this.__mockApp) { + this.__mockApp = express(); + } + const server = this.__mockApp.listen(...args); + return server; +}; + +export { MockExpressListener as Server }; diff --git a/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/package.json b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/package.json new file mode 100644 index 000000000..780fc652a --- /dev/null +++ b/packages/docker-runner/__tests__/mocks/@openziti/ziti-sdk-nodejs/package.json @@ -0,0 +1,6 @@ +{ + "name": "@openziti/ziti-sdk-nodejs", + "version": "0.0.0-test", + "main": "index.js", + "type": "module" +} diff --git a/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs b/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs new file mode 100644 index 000000000..d350becfc --- /dev/null +++ b/packages/docker-runner/__tests__/mocks/mock-openziti-loader.mjs @@ -0,0 +1,24 @@ +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const mockModuleRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '@openziti', + 'ziti-sdk-nodejs', +); + +export async function resolve(specifier, context, defaultResolve) { + if (specifier === '@openziti/ziti-sdk-nodejs') { + return { + shortCircuit: true, + url: pathToFileURL(path.join(mockModuleRoot, 'index.js')).href, + }; + } + if (specifier === '@openziti/ziti-sdk-nodejs/lib/express-listener.js') { + return { + shortCircuit: true, + url: pathToFileURL(path.join(mockModuleRoot, 'lib', 'express-listener.js')).href, + }; + } + return defaultResolve(specifier, context, defaultResolve); +} diff --git a/packages/docker-runner/src/service/main.ts b/packages/docker-runner/src/service/main.ts index 66fe28ab9..473000faf 100644 --- a/packages/docker-runner/src/service/main.ts +++ b/packages/docker-runner/src/service/main.ts @@ -14,6 +14,7 @@ async function bootstrap(): Promise { const config = loadRunnerConfig(); app = createRunnerApp(config); await app.listen({ port: config.port, host: config.host }); + console.info(`docker-runner listening on http://${config.host}:${config.port}`); const ingress = await startZitiIngress(config); closeZiti = ingress.close; diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts index 6e5d9fa50..a6366caba 100644 --- a/packages/docker-runner/src/service/ziti.ingress.ts +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -23,19 +23,16 @@ type ZitiExpressListenerModule = { let zitiExpressPatched = false; export async function startZitiIngress(config: RunnerConfig): Promise { - if (process.env.ZITI_BYPASS === '1') { - console.warn('Ziti ingress bypassed via ZITI_BYPASS=1'); - return { - close: async () => {}, - } satisfies ZitiIngressHandle; - } const ziti = await import('@openziti/ziti-sdk-nodejs'); - const identityPath = config.ziti.identityFile; + const identityPath = ensureIdentityPath(config.ziti.identityFile); + const serviceName = ensureServiceName(config.ziti.serviceName); await ensureIdentityReadable(identityPath); + console.info(`Initializing OpenZiti ingress (identity=${identityPath}, service=${serviceName})`); await ziti.init(identityPath); + console.info(`OpenZiti SDK initialized for service ${serviceName}`); await ensureZitiExpressServerPatch(); - const app = ziti.express(express, config.ziti.serviceName); + const app = ziti.express(express, serviceName); const targetHost = resolveTargetHost(config.host); const target = `http://${targetHost}:${config.port}`; const proxy = httpProxy.createProxyServer({ @@ -56,7 +53,7 @@ export async function startZitiIngress(config: RunnerConfig): Promise socket.destroy()); }); - console.info(`Ziti ingress ready for service ${config.ziti.serviceName}`); + console.info(`Ziti ingress ready for service ${serviceName} (target=${target})`); return { close: async () => { @@ -140,3 +137,19 @@ async function ensureIdentityReadable(file: string): Promise { throw new Error(`Ziti identity file missing or unreadable: ${file}`); } } + +const ensureIdentityPath = (file: string | undefined): string => { + const trimmed = file?.trim(); + if (!trimmed) { + throw new Error('Ziti identity file path missing (ZITI_IDENTITY_FILE)'); + } + return trimmed; +}; + +const ensureServiceName = (service: string | undefined): string => { + const trimmed = service?.trim(); + if (!trimmed) { + throw new Error('Ziti service name missing (ZITI_SERVICE_NAME)'); + } + return trimmed; +}; diff --git a/packages/platform-server/.env.example b/packages/platform-server/.env.example index 44ad7ae42..f9ecb99fb 100644 --- a/packages/platform-server/.env.example +++ b/packages/platform-server/.env.example @@ -43,7 +43,8 @@ VAULT_TOKEN=dev-root WORKSPACE_NETWORK_NAME=agents_net # docker-runner endpoint and credentials (required) -DOCKER_RUNNER_BASE_URL=http://docker-runner:7071 +# Host dev proxy runs on 127.0.0.1:17071 via ConfigService defaults +DOCKER_RUNNER_BASE_URL=http://127.0.0.1:17071 DOCKER_RUNNER_SHARED_SECRET=dev-shared-secret # Optional request timeout override (ms) # DOCKER_RUNNER_TIMEOUT_MS=30000 diff --git a/packages/platform-server/__tests__/helpers/docker.e2e.ts b/packages/platform-server/__tests__/helpers/docker.e2e.ts index b8e92830b..0f6149008 100644 --- a/packages/platform-server/__tests__/helpers/docker.e2e.ts +++ b/packages/platform-server/__tests__/helpers/docker.e2e.ts @@ -50,13 +50,21 @@ export async function startDockerRunnerProcess(socketPath: string): Promise const trimUrl = (value: string): string => value.trim().replace(/\/+$/, ''); +const normalizeOptionalUrl = (value?: string | null): string | undefined => { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + return trimUrl(trimmed); +}; + const defaultZitiConfig = { managementUrl: 'https://127.0.0.1:1280/edge/management/v1', username: 'admin', @@ -105,6 +116,13 @@ export const configSchema = z.object({ const num = typeof v === 'number' ? v : Number(v); return Number.isFinite(num) ? num : 30_000; }), + dockerRunnerBaseUrl: z + .union([z.string(), z.undefined()]) + .transform((value) => normalizeOptionalUrl(value)) + .refine( + (value) => !value || /^https?:\/\//.test(value), + 'DOCKER_RUNNER_BASE_URL must include http:// or https://', + ), // Workspace container network name workspaceNetworkName: z.string().min(1).default('agents_net'), // Nix search/proxy settings @@ -410,6 +428,10 @@ export class ConfigService implements Config { } getDockerRunnerBaseUrl(): string { + const explicit = this.params.dockerRunnerBaseUrl; + if (explicit) { + return explicit; + } const host = this.getZitiRunnerProxyHost(); const port = this.getZitiRunnerProxyPort(); if (!host) { @@ -599,6 +621,7 @@ export class ConfigService implements Config { dockerMirrorUrl: process.env.DOCKER_MIRROR_URL, dockerRunnerSharedSecret: process.env.DOCKER_RUNNER_SHARED_SECRET, dockerRunnerTimeoutMs: process.env.DOCKER_RUNNER_TIMEOUT_MS, + dockerRunnerBaseUrl: process.env.DOCKER_RUNNER_BASE_URL, workspaceNetworkName: process.env.WORKSPACE_NETWORK_NAME, nixAllowedChannels: process.env.NIX_ALLOWED_CHANNELS, nixHttpTimeoutMs: process.env.NIX_HTTP_TIMEOUT_MS, From 53c7a67a8d6877ade5185fbc59d7eb1dfdc2ba1e Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Sat, 21 Feb 2026 06:40:43 +0000 Subject: [PATCH 21/22] fix(ziti): stabilize controller bootstrap --- docker-compose.yml | 8 ++++---- packages/docker-runner/src/service/ziti.ingress.ts | 2 +- .../src/infra/ziti/ziti.management.client.ts | 11 ++++++++++- .../platform-server/src/infra/ziti/ziti.reconciler.ts | 7 ++++++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 723ac5d1a..6727fe9d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: # Dedicated Postgres for agents persistence agents-db: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-public.ecr.aws/docker/library/postgres:16-alpine} oom_score_adj: -900 container_name: agents-db restart: unless-stopped @@ -250,7 +250,7 @@ services: - agents_net ziti-controller: - image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + image: ${ZITI_IMAGE:-mirror.gcr.io/openziti/quickstart}:${ZITI_VERSION:-latest} container_name: ziti-controller hostname: ziti-controller restart: unless-stopped @@ -291,7 +291,7 @@ services: - agents_net ziti-controller-init: - image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + image: ${ZITI_IMAGE:-mirror.gcr.io/openziti/quickstart}:${ZITI_VERSION:-latest} container_name: ziti-controller-init restart: "no" depends_on: @@ -338,7 +338,7 @@ services: - agents_net ziti-edge-router: - image: ${ZITI_IMAGE:-openziti/quickstart}:${ZITI_VERSION:-latest} + image: ${ZITI_IMAGE:-mirror.gcr.io/openziti/quickstart}:${ZITI_VERSION:-latest} container_name: ziti-edge-router hostname: ziti-edge-router restart: unless-stopped diff --git a/packages/docker-runner/src/service/ziti.ingress.ts b/packages/docker-runner/src/service/ziti.ingress.ts index a6366caba..e8bc97bf0 100644 --- a/packages/docker-runner/src/service/ziti.ingress.ts +++ b/packages/docker-runner/src/service/ziti.ingress.ts @@ -133,7 +133,7 @@ async function ensureZitiExpressServerPatch(): Promise { async function ensureIdentityReadable(file: string): Promise { try { await fs.access(file, fsConstants.R_OK); - } catch (error) { + } catch (_error) { throw new Error(`Ziti identity file missing or unreadable: ${file}`); } } diff --git a/packages/platform-server/src/infra/ziti/ziti.management.client.ts b/packages/platform-server/src/infra/ziti/ziti.management.client.ts index c04f244ee..cbc803abe 100644 --- a/packages/platform-server/src/infra/ziti/ziti.management.client.ts +++ b/packages/platform-server/src/infra/ziti/ziti.management.client.ts @@ -196,6 +196,13 @@ export class ZitiManagementClient { const response = await this.request>('GET', path, { searchParams: { filter }, }); + if (process.env.ZITI_DEBUG === '1') { + this.logger.debug( + `findByName path=${path} filter=${filter} count=${response.data?.length ?? 0} raw=${JSON.stringify( + response, + )}`, + ); + } return response.data?.[0]; } @@ -205,7 +212,9 @@ export class ZitiManagementClient { } private async request(method: string, path: string, options: RequestOptions = {}): Promise { - const url = new URL(path, this.options.baseUrl); + const sanitizedPath = path.replace(/^\/+/, ''); + const baseUrl = this.options.baseUrl.endsWith('/') ? this.options.baseUrl : `${this.options.baseUrl}/`; + const url = new URL(sanitizedPath, baseUrl); if (options.searchParams) { for (const [key, value] of Object.entries(options.searchParams)) { if (typeof value === 'string' && value.length > 0) { diff --git a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts index 04901d7d0..fc0733149 100644 --- a/packages/platform-server/src/infra/ziti/ziti.reconciler.ts +++ b/packages/platform-server/src/infra/ziti/ziti.reconciler.ts @@ -36,6 +36,7 @@ export class ZitiReconciler { ) {} async reconcile(): Promise { + this.logger.log('Reconciling Ziti control-plane'); const profile = this.buildProfile(); const client = new ZitiManagementClient({ baseUrl: profile.managementUrl, @@ -70,6 +71,7 @@ export class ZitiReconciler { directories: profile.directories, client, }); + this.logger.log('Ziti identities reconciled'); } finally { await client.close(); } @@ -139,13 +141,16 @@ export class ZitiReconciler { try { const router = await client.getEdgeRouterByName(profile.routerName); if (router) { + this.logger.debug(`Ziti router ${profile.routerName} found`); return router; } lastError = new Error(`router ${profile.routerName} not yet registered`); } catch (error) { lastError = error as Error; } - + this.logger.debug( + `Waiting for router ${profile.routerName}, lastError=${lastError?.message ?? 'none'} (next poll in ${ROUTER_DISCOVERY_INTERVAL_MS / 1000}s)`, + ); this.logger.log( `Waiting for Ziti router ${profile.routerName} to register (retrying in ${ROUTER_DISCOVERY_INTERVAL_MS / 1000}s)`, ); From e68af37ff4dae882a83fccda3801326e740a6606 Mon Sep 17 00:00:00 2001 From: Casey Brooks Date: Sat, 21 Feb 2026 13:17:45 +0000 Subject: [PATCH 22/22] chore(ziti): harden e2e stack deps --- docs/containers/ziti.md | 22 ++ e2e/ziti/docker-compose.ci.yml | 355 ++++++++++++++++++++++ e2e/ziti/run-e2e.mjs | 472 ++++++++++++++++++++++++++++++ packages/docker-runner/Dockerfile | 16 +- 4 files changed, 864 insertions(+), 1 deletion(-) create mode 100644 e2e/ziti/docker-compose.ci.yml create mode 100644 e2e/ziti/run-e2e.mjs diff --git a/docs/containers/ziti.md b/docs/containers/ziti.md index c61b6db60..3ae53d8af 100644 --- a/docs/containers/ziti.md +++ b/docs/containers/ziti.md @@ -21,6 +21,16 @@ pnpm ziti:prepare 2. Install dependencies and explicitly allow the OpenZiti SDK build step (pnpm blocks install scripts by default): +> **Linux build prerequisites** +> +> The OpenZiti Node SDK falls back to a full native build whenever a prebuilt binary +> is unavailable (for example, when working from the agyn fork). Make sure the host +> has the standard build toolchain plus `autoconf`, `automake`, `libtool`, `m4`, and +> `perl` alongside the existing `build-essential`, `cmake`, `ninja-build`, +> `python3`, `pkg-config`, `git`, `curl`, `zip`, and `unzip` packages. The CI +> containers install the same list so the docker-runner/platform-server bring-up can +> compile the SDK reliably. + ```bash pnpm approve-builds # Select @openziti/ziti-sdk-nodejs and confirm @@ -86,6 +96,18 @@ ZITI_SERVICE_NAME=dev.agyn-platform.platform-api > Replace `/absolute/path/to/platform` with your local repository root (for example `/Users/casey/dev/platform`). +## CI-aligned smoke test + +- Run `pnpm --filter @agyn/platform-server run test:ziti` after the prerequisites above to boot the same lean stack used + by CI (`e2e/ziti/docker-compose.ci.yml`). +- The helper script wipes `.ziti`, brings the controller/router/runner online, ensures the DinD engine exposes the + `agents_net` network, and drives a real workspace create → delete cycle via HTTP. +- No container builds occur; the Node containers mount the local checkout and reuse the existing `node_modules` tree so + the loop completes in under five minutes. +- Logs for `ziti-controller`, `ziti-edge-router`, `docker-runner`, and `platform-server` are dumped automatically on + failure to speed up triage. +- The flow enables the private `/test/workspaces` controller via `ENABLE_TEST_WORKSPACE_API=1`, matching the CI job. + ## Host-mode workflow After completing the prerequisites and enabling the `.env` entries above, the developer stack can be verified on the host with the following sequence (clean-room friendly): diff --git a/e2e/ziti/docker-compose.ci.yml b/e2e/ziti/docker-compose.ci.yml new file mode 100644 index 000000000..b6f433093 --- /dev/null +++ b/e2e/ziti/docker-compose.ci.yml @@ -0,0 +1,355 @@ +version: '3.9' + +services: + agents-db: + image: public.ecr.aws/docker/library/postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: agents + POSTGRES_USER: agents + POSTGRES_PASSWORD: agents + ports: + - '127.0.0.1:5443:5432' + volumes: + - agents_pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U agents -d agents'] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + networks: + - agents_net + + litellm-db: + image: public.ecr.aws/docker/library/postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: change-me + volumes: + - litellm_pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U litellm -d litellm'] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + networks: + - agents_net + + litellm: + image: ghcr.io/berriai/litellm:v1.80.5-stable + restart: unless-stopped + environment: + LITELLM_MASTER_KEY: sk-litellm-ci + LITELLM_SALT_KEY: sk-litellm-salt + DATABASE_URL: postgresql://litellm:change-me@litellm-db:5432/litellm + STORE_MODEL_IN_DB: 'True' + UI_USERNAME: admin + UI_PASSWORD: admin + PORT: '4000' + HOST: '0.0.0.0' + depends_on: + litellm-db: + condition: service_healthy + ports: + - '127.0.0.1:4000:4000' + healthcheck: + test: + [ + 'CMD', + 'python3', + '-c', + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/ui')", + ] + interval: 15s + timeout: 3s + retries: 5 + start_period: 10s + networks: + - agents_net + + ziti-controller: + image: openziti/quickstart:1.7.0 + hostname: ziti-controller + restart: unless-stopped + user: '0:0' + entrypoint: + - '/var/openziti/scripts/run-controller.sh' + environment: + ZITI_NETWORK: dev.agyn-platform + ZITI_CTRL_NAME: dev-controller + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: '1280' + ZITI_CTRL_EDGE_IP_OVERRIDE: 127.0.0.1 + ZITI_CTRL_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_ADVERTISED_PORT: '6262' + ZITI_EDGE_IDENTITY_ENROLLMENT_DURATION: 168h + ZITI_ROUTER_ENROLLMENT_DURATION: 168h + ZITI_USER: admin + ZITI_PWD: admin + ports: + - '127.0.0.1:1280:1280' + - '127.0.0.1:6262:6262' + healthcheck: + test: + [ + 'CMD-SHELL', + 'curl -m 1 -s -k -f https://ziti-controller:1280/edge/client/v1/version', + ] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - type: bind + source: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/controller + target: /persistent + bind: + create_host_path: true + networks: + - agents_net + + ziti-controller-init: + image: openziti/quickstart:1.7.0 + restart: 'no' + user: '0:0' + depends_on: + ziti-controller: + condition: service_healthy + entrypoint: + - '/bin/bash' + - '/scripts/ziti/controller-init.sh' + environment: + ZITI_NETWORK: dev.agyn-platform + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: '1280' + ZITI_USER: admin + ZITI_PWD: admin + ZITI_SERVICE_NAME: dev.agyn-platform.platform-api + ZITI_PLATFORM_IDENTITY_NAME: dev.agyn-platform.platform-server + ZITI_RUNNER_IDENTITY_NAME: dev.agyn-platform.docker-runner + ZITI_ROUTER_NAME: dev-edge-router + ZITI_ENROLLMENT_DURATION_MINUTES: '1440' + ZITI_IDENTITIES_DIR: /identities + ZITI_IDENTITIES_TMP: /ziti-tmp + ZITI_PLATFORM_IDENTITY_FILE: /identities/dev.agyn-platform.platform-server.json + ZITI_RUNNER_IDENTITY_FILE: /identities/dev.agyn-platform.docker-runner.json + volumes: + - type: bind + source: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/controller + target: /persistent + bind: + create_host_path: true + - type: bind + source: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/identities + target: /identities + bind: + create_host_path: true + - type: bind + source: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/tmp + target: /ziti-tmp + bind: + create_host_path: true + - type: bind + source: ${ZITI_E2E_ROOT}/scripts/ziti + target: /scripts/ziti + networks: + - agents_net + + ziti-edge-router: + image: openziti/quickstart:1.7.0 + hostname: ziti-edge-router + restart: unless-stopped + user: '0:0' + depends_on: + ziti-controller: + condition: service_healthy + entrypoint: /bin/bash + command: '/var/openziti/scripts/run-router.sh edge' + environment: + ZITI_NETWORK: dev.agyn-platform + ZITI_CTRL_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_ADVERTISED_PORT: '6262' + ZITI_CTRL_EDGE_ADVERTISED_ADDRESS: ziti-controller + ZITI_CTRL_EDGE_ADVERTISED_PORT: '1280' + ZITI_ROUTER_NAME: dev-edge-router + ZITI_ROUTER_ADVERTISED_ADDRESS: ziti-edge-router + ZITI_ROUTER_PORT: '3022' + ZITI_ROUTER_LISTENER_BIND_PORT: '10080' + ZITI_ROUTER_ROLES: router.platform + ZITI_USER: admin + ZITI_PWD: admin + ports: + - '127.0.0.1:3022:3022' + volumes: + - type: bind + source: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/controller + target: /persistent + bind: + create_host_path: true + networks: + - agents_net + + dind: + image: public.ecr.aws/docker/library/docker:26.1-dind + privileged: true + environment: + DOCKER_TLS_CERTDIR: '' + command: + - '--host=tcp://0.0.0.0:2375' + - '--host=unix:///var/run/docker.sock' + healthcheck: + test: ['CMD-SHELL', 'docker info >/dev/null 2>&1'] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - docker_dind_data:/var/lib/docker + - docker_dind_run:/var/run + networks: + - agents_net + + docker-runner: + image: public.ecr.aws/docker/library/node:20.18.0-bookworm + working_dir: /workspace/platform + command: >- + bash -lc "set -euo pipefail; + apt-get update >/tmp/apt.log && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake ninja-build python3 pkg-config git curl zip unzip autoconf automake libtool m4 perl >>/tmp/apt.log; + mkdir -p /workspace/platform/.cache; + if [ ! -d /workspace/platform/.cache/vcpkg/.git ]; then + rm -rf /workspace/platform/.cache/vcpkg; + git clone https://github.com/microsoft/vcpkg.git /workspace/platform/.cache/vcpkg >>/tmp/vcpkg-bootstrap.log; + cd /workspace/platform/.cache/vcpkg && git checkout 84bab45d415d22042bd0b9081aea57f362da3f35 >>/tmp/vcpkg-bootstrap.log; + fi; + if [ ! -x /workspace/platform/.cache/vcpkg/vcpkg ]; then + cd /workspace/platform/.cache/vcpkg && ./bootstrap-vcpkg.sh -disableMetrics >>/tmp/vcpkg-bootstrap.log; + fi; + export VCPKG_ROOT=/workspace/platform/.cache/vcpkg; + if ! command -v pnpm >/dev/null 2>&1 || [ \"$(pnpm --version 2>/dev/null)\" != \"10.5.0\" ]; then + npm install -g pnpm@10.5.0 > /tmp/install-pnpm.log; + fi; + pnpm install --config.allow-scripts=true --frozen-lockfile --prefer-offline; + pnpm rebuild @nestjs/core @openziti/ziti-sdk-nodejs @prisma/client @prisma/engines @swc/core @tailwindcss/oxide cpu-features esbuild msw prisma protobufjs ssh2 unrs-resolver; + pnpm --filter @agyn/docker-runner dev" + environment: + NODE_ENV: production + DOCKER_RUNNER_HOST: 0.0.0.0 + DOCKER_RUNNER_PORT: '7071' + DOCKER_RUNNER_SHARED_SECRET: ci-ziti-runner + DOCKER_RUNNER_LOG_LEVEL: warn + DOCKER_SOCKET: /var/run/docker.sock + ZITI_IDENTITY_FILE: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/identities/dev.agyn-platform.docker-runner.json + ZITI_SERVICE_NAME: dev.agyn-platform.platform-api + ZITI_LOG_LEVEL: ${ZITI_LOG_LEVEL:-} + ZITI_SDK_DIR: /workspace/platform/packages/ziti-sdk-c + VCPKG_ROOT: /workspace/platform/.cache/vcpkg + depends_on: + dind: + condition: service_healthy + ziti-controller-init: + condition: service_completed_successfully + ziti-edge-router: + condition: service_started + healthcheck: + test: ['CMD-SHELL', 'curl -fsS http://127.0.0.1:7071/v1/ready || exit 1'] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + volumes: + - type: bind + source: ${ZITI_E2E_ROOT} + target: /workspace/platform + - docker_dind_run:/var/run + networks: + - agents_net + + platform-server: + image: public.ecr.aws/docker/library/node:20.18.0-bookworm + working_dir: /workspace/platform + command: >- + bash -lc "set -euo pipefail; + mkdir -p /workspace/platform/.tmp/graph-ziti-e2e; + apt-get update >/tmp/apt.log && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake ninja-build python3 pkg-config git curl zip unzip autoconf automake libtool m4 perl >>/tmp/apt.log; + mkdir -p /workspace/platform/.cache; + if [ ! -d /workspace/platform/.cache/vcpkg/.git ]; then + rm -rf /workspace/platform/.cache/vcpkg; + git clone https://github.com/microsoft/vcpkg.git /workspace/platform/.cache/vcpkg >>/tmp/vcpkg-bootstrap.log; + cd /workspace/platform/.cache/vcpkg && git checkout 84bab45d415d22042bd0b9081aea57f362da3f35 >>/tmp/vcpkg-bootstrap.log; + fi; + if [ ! -x /workspace/platform/.cache/vcpkg/vcpkg ]; then + cd /workspace/platform/.cache/vcpkg && ./bootstrap-vcpkg.sh -disableMetrics >>/tmp/vcpkg-bootstrap.log; + fi; + export VCPKG_ROOT=/workspace/platform/.cache/vcpkg; + if ! command -v pnpm >/dev/null 2>&1 || [ \"$(pnpm --version 2>/dev/null)\" != \"10.5.0\" ]; then + npm install -g pnpm@10.5.0 > /tmp/install-pnpm.log; + fi; + pnpm install --config.allow-scripts=true --frozen-lockfile --prefer-offline; + pnpm rebuild @nestjs/core @openziti/ziti-sdk-nodejs @prisma/client @prisma/engines @swc/core @tailwindcss/oxide cpu-features esbuild msw prisma protobufjs ssh2 unrs-resolver; + pnpm --filter @agyn/platform-server exec prisma migrate deploy; + pnpm --filter @agyn/platform-server dev" + environment: + NODE_ENV: production + PORT: '3010' + AGENTS_DATABASE_URL: postgresql://agents:agents@agents-db:5432/agents + LLM_PROVIDER: litellm + LITELLM_BASE_URL: http://litellm:4000 + LITELLM_MASTER_KEY: sk-litellm-ci + GRAPH_REPO_PATH: ${ZITI_E2E_ROOT}/.tmp/graph-ziti-e2e + WORKSPACE_NETWORK_NAME: agents_net + DOCKER_RUNNER_SHARED_SECRET: ci-ziti-runner + DOCKER_RUNNER_BASE_URL: http://127.0.0.1:17071 + ZITI_MANAGEMENT_URL: https://ziti-controller:1280/edge/management/v1 + ZITI_USERNAME: admin + ZITI_PASSWORD: admin + ZITI_INSECURE_TLS: 'true' + ZITI_SERVICE_NAME: dev.agyn-platform.platform-api + ZITI_ROUTER_NAME: dev-edge-router + ZITI_RUNNER_PROXY_HOST: 0.0.0.0 + ZITI_RUNNER_PROXY_PORT: '17071' + ZITI_PLATFORM_IDENTITY_FILE: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/identities/dev.agyn-platform.platform-server.json + ZITI_RUNNER_IDENTITY_FILE: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/identities/dev.agyn-platform.docker-runner.json + ZITI_IDENTITIES_DIR: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/identities + ZITI_TMP_DIR: ${ZITI_E2E_STATE_DIR:-${ZITI_E2E_ROOT}/.ziti}/tmp + ZITI_LOG_LEVEL: ${ZITI_LOG_LEVEL:-} + ZITI_SDK_DIR: /workspace/platform/packages/ziti-sdk-c + VCPKG_ROOT: /workspace/platform/.cache/vcpkg + ENABLE_TEST_WORKSPACE_API: '1' + PRISMA_HIDE_UPDATE_MESSAGE: '1' + depends_on: + agents-db: + condition: service_healthy + litellm: + condition: service_healthy + docker-runner: + condition: service_healthy + ziti-controller-init: + condition: service_completed_successfully + ziti-edge-router: + condition: service_started + ports: + - '127.0.0.1:3010:3010' + healthcheck: + test: ['CMD-SHELL', 'curl -fsS http://127.0.0.1:3010/api/containers?limit=1 || exit 1'] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + volumes: + - type: bind + source: ${ZITI_E2E_ROOT} + target: /workspace/platform + networks: + - agents_net + +volumes: + agents_pgdata: + litellm_pgdata: + docker_dind_data: + docker_dind_run: + +networks: + agents_net: + name: agents_net diff --git a/e2e/ziti/run-e2e.mjs b/e2e/ziti/run-e2e.mjs new file mode 100644 index 000000000..c1db278d5 --- /dev/null +++ b/e2e/ziti/run-e2e.mjs @@ -0,0 +1,472 @@ +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { waitForServices } from './wait-for-services.mjs'; + +const ROOT_DIR = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); +const COMPOSE_FILE = path.join(ROOT_DIR, 'e2e/ziti/docker-compose.ci.yml'); +const PROJECT_NAME = process.env.ZITI_E2E_PROJECT ?? 'ziti_e2e'; +const HOME_DIR = process.env.HOME ?? os.homedir(); +const HOME_DOCKER_CONFIG = path.join(HOME_DIR, '.docker'); +const WORKSPACE_DOCKER_CONFIG = path.resolve(ROOT_DIR, '..', '.docker'); +const HOST_WORKSPACE_ROOT = '/workspace/platform'; +const HOST_CONTAINER_WORKSPACE = path.posix.join('/host', path.posix.relative('/workspace', HOST_WORKSPACE_ROOT)); +const STATE_DIR = path.join(ROOT_DIR, '.ziti'); +process.env.ZITI_E2E_STATE_DIR = STATE_DIR; +const UTILITY_IMAGE = process.env.ZITI_E2E_UTILITY_IMAGE ?? 'public.ecr.aws/docker/library/alpine:3.20'; +const STATE_PATHS = { + root: STATE_DIR, + controller: path.join(STATE_DIR, 'controller'), + identities: path.join(STATE_DIR, 'identities'), + tmp: path.join(STATE_DIR, 'tmp'), +}; +const DEFAULT_DOCKER_CONFIG = + process.env.DOCKER_CONFIG + ?? (hasComposePlugin(HOME_DOCKER_CONFIG) ? HOME_DOCKER_CONFIG : undefined) + ?? (hasComposePlugin(WORKSPACE_DOCKER_CONFIG) ? WORKSPACE_DOCKER_CONFIG : undefined) + ?? HOME_DOCKER_CONFIG; +const MAIN_TIMEOUT_MS = Number(process.env.ZITI_E2E_TIMEOUT_MS ?? 300_000); +const REQUEST_TIMEOUT_MS = 15_000; + +if (!process.env.DOCKER_CONFIG) { + process.env.DOCKER_CONFIG = DEFAULT_DOCKER_CONFIG; +} + +console.log('[ziti-e2e] DOCKER_CONFIG', process.env.DOCKER_CONFIG); +console.log('[ziti-e2e] state dir', STATE_DIR); + +const SERVICES_TO_WAIT = [ + { name: 'agents-db', requireHealth: true }, + { name: 'litellm-db', requireHealth: true }, + { name: 'litellm', requireHealth: true }, + { name: 'ziti-controller', requireHealth: true }, + { name: 'ziti-controller-init', completed: true }, + { name: 'ziti-edge-router', requireHealth: false }, + { name: 'dind', requireHealth: true }, + { name: 'docker-runner', requireHealth: true }, + { name: 'platform-server', requireHealth: true }, +]; + +main().catch((error) => { + console.error('[ziti-e2e] failed', error); + process.exitCode = 1; +}); + +async function main() { + await withTimeout(runFlow(), MAIN_TIMEOUT_MS, `Ziti E2E timed out after ${MAIN_TIMEOUT_MS}ms`); +} + +async function runFlow() { + await ensureWorkspaceAvailableOnDockerHost(); + const composeRunner = await createComposeRunner(); + registerSignalHandlers(composeRunner); + await composeRunner.down({ quiet: true }); + try { + await composeRunner.run( + ['up', '-d', 'agents-db', 'litellm-db', 'litellm', 'ziti-controller', 'ziti-controller-init', 'ziti-edge-router', 'dind', 'docker-runner', 'platform-server'], + { streamOutput: true }, + ); + await waitForServices({ + compose: composeRunner.run, + inspect: (containerId) => runCommand('docker', ['inspect', containerId], { cwd: ROOT_DIR }), + services: SERVICES_TO_WAIT, + timeoutMs: 180_000, + }); + await ensureWorkspaceNetwork(composeRunner.run); + await waitForHttpReady('http://127.0.0.1:3010/api/containers?limit=1'); + await runWorkspaceLifecycle(); + } catch (error) { + await dumpDiagnostics(composeRunner.run); + throw error; + } finally { + if (process.env.ZITI_E2E_KEEP_STACK === '1') { + console.warn('[ziti-e2e] skipping docker compose down (ZITI_E2E_KEEP_STACK=1)'); + } else { + await composeRunner.down(); + } + } +} + +async function createComposeRunner() { + const binary = await detectComposeBinary(); + const env = { + ...process.env, + DOCKER_CONFIG: DEFAULT_DOCKER_CONFIG, + COMPOSE_PROJECT_NAME: PROJECT_NAME, + ZITI_E2E_ROOT: ROOT_DIR, + ZITI_E2E_STATE_DIR: STATE_DIR, + }; + + const run = async (args, options = {}) => { + const fullArgs = [...binary.args, '-f', COMPOSE_FILE, ...args]; + return runCommand(binary.command, fullArgs, { + cwd: ROOT_DIR, + env, + streamOutput: options.streamOutput ?? false, + quiet: options.quiet ?? false, + }); + }; + + const down = async (options = {}) => { + try { + await run(['down', '-v', '--remove-orphans'], { streamOutput: options.streamOutput, quiet: options.quiet }); + } catch (error) { + if (!(options.quiet ?? false)) { + console.warn('[ziti-e2e] compose down failed', error); + } + } + }; + + return { command: binary.command, run, down }; +} + +async function detectComposeBinary() { + const dockerEnv = { ...process.env, DOCKER_CONFIG: DEFAULT_DOCKER_CONFIG }; + try { + await runCommand('docker', ['compose', 'version'], { quiet: true, env: dockerEnv }); + return { command: 'docker', args: ['compose'] }; + } catch (error) { + console.warn('[ziti-e2e] docker compose plugin unavailable, falling back to docker-compose'); + if (error?.stderr) { + console.warn(error.stderr); + } + await runCommand('docker-compose', ['version'], { quiet: true }); + return { command: 'docker-compose', args: [] }; + } +} + +function registerSignalHandlers(composeRunner) { + let cleaning = false; + const handler = async (signal) => { + if (cleaning) { + return; + } + cleaning = true; + console.warn(`[ziti-e2e] received ${signal}, stopping stack...`); + await composeRunner.down({ quiet: true }); + process.exit(1); + }; + process.once('SIGINT', handler); + process.once('SIGTERM', handler); +} + +async function prepareLocalZitiDirectories() { + await fs.mkdir(STATE_PATHS.root, { recursive: true }); + await fs.chmod(STATE_PATHS.root, 0o777); + const dirs = ['controller', 'identities', 'tmp']; + for (const dir of dirs) { + const target = STATE_PATHS[dir]; + await fs.rm(target, { recursive: true, force: true }); + await fs.mkdir(target, { recursive: true }); + await fs.chmod(target, 0o777); + } +} + +async function verifyZitiDirectories() { + const dirs = ['controller', 'identities', 'tmp']; + for (const dir of dirs) { + const target = STATE_PATHS[dir]; + try { + await fs.stat(target); + } catch (error) { + throw new Error(`Missing Ziti directory: ${target} (${error.message})`); + } + } +} + +async function ensureWorkspaceAvailableOnDockerHost() { + const shared = await isWorkspaceSharedWithDockerHost(); + if (shared) { + console.log('[ziti-e2e] docker host shares workspace; preparing local .ziti state'); + await prepareLocalZitiDirectories(); + await verifyZitiDirectories(); + return; + } + console.log('[ziti-e2e] docker host cannot read workspace directly; syncing repository snapshot'); + await syncWorkspaceToDockerHost(); +} + +async function isWorkspaceSharedWithDockerHost() { + const probeName = `.ziti-host-probe-${Date.now()}`; + const probePath = path.join(ROOT_DIR, probeName); + await fs.writeFile(probePath, 'probe', 'utf8'); + try { + await runCommand('docker', ['run', '--rm', '-v', `${ROOT_DIR}:/host`, UTILITY_IMAGE, 'test', '-f', `/host/${probeName}`], { + quiet: true, + }); + return true; + } catch { + return false; + } finally { + await fs.rm(probePath, { force: true }).catch(() => {}); + } +} + +async function syncWorkspaceToDockerHost() { + const containerName = `${PROJECT_NAME}_host_sync`; + await runCommand('docker', ['rm', '-f', containerName], { quiet: true }).catch(() => {}); + await runCommand('docker', ['run', '-d', '--name', containerName, '-v', '/workspace:/host', UTILITY_IMAGE, 'sleep', '3600'], { + quiet: true, + }); + try { + await runCommand( + 'docker', + ['exec', containerName, 'sh', '-c', `rm -rf ${HOST_CONTAINER_WORKSPACE} && mkdir -p ${HOST_CONTAINER_WORKSPACE}`], + { quiet: true }, + ); + await streamRepoToHost(containerName); + const initScript = [ + `mkdir -p ${HOST_CONTAINER_WORKSPACE}/.ziti/controller ${HOST_CONTAINER_WORKSPACE}/.ziti/identities ${HOST_CONTAINER_WORKSPACE}/.ziti/tmp`, + `chmod -R 0777 ${HOST_CONTAINER_WORKSPACE}/.ziti`, + ].join(' && '); + await runCommand('docker', ['exec', containerName, 'sh', '-c', initScript], { quiet: true }); + } finally { + await runCommand('docker', ['rm', '-f', containerName], { quiet: true }).catch(() => {}); + } +} + +function streamRepoToHost(containerName) { + return new Promise((resolve, reject) => { + const excludes = ['.git', 'node_modules', '.turbo', '.tmp', '.ziti', '.pnpm-store', '.cache', 'coverage']; + const tarArgs = [ + '-C', + ROOT_DIR, + ...excludes.flatMap((pattern) => ['--exclude', pattern]), + '-cf', + '-', + '.', + ]; + console.log('[ziti-e2e] syncing repository snapshot to docker host'); + const tarProc = spawn('tar', tarArgs, { stdio: ['ignore', 'pipe', 'inherit'] }); + const dockerProc = spawn('docker', ['exec', '-i', containerName, 'sh', '-c', `cd ${HOST_CONTAINER_WORKSPACE} && tar xf -`], { + quiet: true, + stdio: ['pipe', 'inherit', 'inherit'], + }); + tarProc.stdout.pipe(dockerProc.stdin); + let settled = false; + const cleanup = (error) => { + if (settled) { + return; + } + settled = true; + tarProc.stdout.unpipe(dockerProc.stdin); + dockerProc.stdin.end(); + if (error) { + reject(error); + } else { + resolve(); + } + }; + tarProc.on('error', cleanup); + tarProc.on('close', (code) => { + if (code !== 0) { + cleanup(new Error(`tar exited with code ${code}`)); + } + }); + dockerProc.on('error', cleanup); + dockerProc.on('close', (code) => { + if (code === 0) { + cleanup(); + } else { + cleanup(new Error(`docker exec tar exited with code ${code}`)); + } + }); + }); +} + +async function ensureWorkspaceNetwork(compose) { + try { + await compose(['exec', '-T', 'dind', 'docker', 'network', 'inspect', 'agents_net'], { quiet: true }); + } catch { + await compose(['exec', '-T', 'dind', 'docker', 'network', 'create', '--driver', 'bridge', 'agents_net'], { + streamOutput: true, + }); + } +} + +async function waitForHttpReady(url) { + await waitUntil(async () => { + try { + const response = await fetchWithTimeout(url, { method: 'GET' }); + return response.ok; + } catch { + return false; + } + }, { + timeoutMs: 120_000, + intervalMs: 2_000, + description: 'platform-server readiness', + }); +} + +async function runWorkspaceLifecycle() { + console.log('[ziti-e2e] creating workspace'); + const { containerId, threadId } = await createWorkspace(); + console.log(`[ziti-e2e] workspace created container=${containerId} thread=${threadId}`); + await waitUntil(async () => { + const state = await findContainer(containerId, false); + return state?.status === 'running'; + }, { timeoutMs: 60_000, intervalMs: 2_000, description: 'workspace running state' }); + + console.log('[ziti-e2e] deleting workspace'); + await deleteWorkspace(containerId); + await waitUntil(async () => { + const state = await findContainer(containerId, true); + return !!state && state.status === 'stopped'; + }, { timeoutMs: 90_000, intervalMs: 2_000, description: 'workspace cleanup state' }); + console.log('[ziti-e2e] workspace lifecycle verified'); +} + +async function createWorkspace() { + const response = await fetchWithTimeout('http://127.0.0.1:3010/test/workspaces', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ alias: 'ziti-e2e' }), + }); + if (!response.ok) { + throw new Error(`Failed to create workspace (${response.status})`); + } + const payload = await response.json(); + if (!payload?.containerId || !payload?.threadId) { + throw new Error('Workspace response missing containerId or threadId'); + } + return payload; +} + +async function deleteWorkspace(containerId) { + const response = await fetchWithTimeout(`http://127.0.0.1:3010/api/containers/${containerId}`, { + method: 'DELETE', + }); + if (response.status !== 204) { + throw new Error(`Failed to delete workspace container (${response.status})`); + } +} + +async function findContainer(containerId, includeStopped) { + const url = new URL('http://127.0.0.1:3010/api/containers'); + url.searchParams.set('limit', '200'); + if (includeStopped) { + url.searchParams.set('includeStopped', 'true'); + } + const response = await fetchWithTimeout(url, { method: 'GET' }); + if (!response.ok) { + throw new Error(`Failed to list containers (${response.status})`); + } + const payload = await response.json(); + return Array.isArray(payload?.items) + ? payload.items.find((item) => item.containerId === containerId) + : undefined; +} + +async function dumpDiagnostics(compose) { + console.warn('[ziti-e2e] collecting diagnostics'); + try { + await compose(['ps'], { streamOutput: true, quiet: true }); + await compose( + [ + 'logs', + '--tail', + '200', + 'ziti-controller', + 'ziti-controller-init', + 'ziti-edge-router', + 'docker-runner', + 'platform-server', + ], + { streamOutput: true, quiet: true }, + ); + } catch (error) { + console.warn('[ziti-e2e] failed to collect diagnostics', error); + } +} + +async function fetchWithTimeout(resource, init = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS).unref(); + try { + const response = await fetch(resource, { ...init, signal: controller.signal }); + return response; + } finally { + clearTimeout(timeout); + } +} + +async function waitUntil(check, options) { + const { timeoutMs, intervalMs, description } = options; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) { + return; + } + await delay(intervalMs, { ref: false }); + } + throw new Error(`Timeout waiting for ${description ?? 'condition'}`); +} + +async function withTimeout(promise, timeoutMs, message) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), timeoutMs).unref(); + }), + ]); + } finally { + if (timer) { + clearTimeout(timer); + } + } +} + +function runCommand(command, args, options = {}) { + return new Promise((resolve, reject) => { + if (!(options.quiet ?? false)) { + console.log(`[ziti-e2e] > ${command} ${args.join(' ')}`); + } + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (chunk) => { + stdout += chunk; + if (options.streamOutput) { + process.stdout.write(chunk); + } + }); + child.stderr?.on('data', (chunk) => { + stderr += chunk; + if (options.streamOutput) { + process.stderr.write(chunk); + } + }); + child.on('error', (error) => reject(error)); + child.on('close', (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + const err = new Error(`${command} ${args.join(' ')} exited with code ${code}`); + err.stdout = stdout; + err.stderr = stderr; + reject(err); + } + }); + }); +} + +function hasComposePlugin(dir) { + if (!dir) { + return false; + } + const pluginPath = path.join(dir, 'cli-plugins', 'docker-compose'); + return existsSync(pluginPath); +} diff --git a/packages/docker-runner/Dockerfile b/packages/docker-runner/Dockerfile index 83eab6b60..4b2a30a3c 100644 --- a/packages/docker-runner/Dockerfile +++ b/packages/docker-runner/Dockerfile @@ -10,7 +10,21 @@ RUN corepack enable \ && corepack prepare pnpm@10.5.0 --activate RUN apt-get update \ - && apt-get install -y --no-install-recommends git \ + && apt-get install -y --no-install-recommends \ + autoconf \ + automake \ + build-essential \ + cmake \ + curl \ + git \ + libtool \ + m4 \ + ninja-build \ + perl \ + pkg-config \ + python3 \ + unzip \ + zip \ && rm -rf /var/lib/apt/lists/* WORKDIR /workspace