From 249015132dc8d815731d96523dc01629672645d6 Mon Sep 17 00:00:00 2001 From: Andreas Jansson Date: Sun, 1 Feb 2026 23:41:16 +0100 Subject: [PATCH 1/2] Add E2E tests for device pairing and conversation flow - Add E2E test infrastructure with setup/teardown scripts - Add test fixtures: start-server, stop-server, start-browser, stop-browser, pw wrapper - Add pairing_and_conversation test that: - Starts the moltworker server with wrangler dev - Opens browser with playwright-cli - Tests device pairing flow via admin UI - Tests conversation with Claude (math question) - Add video recording support using playwright-cli video-start/video-stop - Add E2E_TEST_MODE to skip CF Access auth during tests - Add isE2ETestMode helper and tests --- .dev.vars.example | 4 + .gitignore | 9 ++ src/auth/middleware.test.ts | 35 ++++++- src/auth/middleware.ts | 13 ++- src/index.ts | 84 +++++++-------- src/types.ts | 1 + test/e2e/_setup.txt | 26 +++++ test/e2e/_teardown.txt | 46 +++++++++ test/e2e/fixture/pw | 28 +++++ test/e2e/fixture/start-browser | 27 +++++ test/e2e/fixture/start-server | 142 ++++++++++++++++++++++++++ test/e2e/fixture/stop-browser | 8 ++ test/e2e/fixture/stop-server | 37 +++++++ test/e2e/pairing_and_conversation.txt | 97 ++++++++++++++++++ 14 files changed, 513 insertions(+), 44 deletions(-) create mode 100644 test/e2e/_setup.txt create mode 100644 test/e2e/_teardown.txt create mode 100755 test/e2e/fixture/pw create mode 100755 test/e2e/fixture/start-browser create mode 100755 test/e2e/fixture/start-server create mode 100755 test/e2e/fixture/stop-browser create mode 100755 test/e2e/fixture/stop-server create mode 100644 test/e2e/pairing_and_conversation.txt diff --git a/.dev.vars.example b/.dev.vars.example index 6d1681172..757ba58b8 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -6,6 +6,10 @@ ANTHROPIC_API_KEY=sk-ant-... # Local development mode - skips Cloudflare Access auth and bypasses device pairing # DEV_MODE=true +# E2E test mode - skips Cloudflare Access auth but keeps device pairing enabled +# Use this for automated tests that need to test the real pairing flow +# E2E_TEST_MODE=true + # Enable debug routes at /debug/* (optional) # DEBUG_ROUTES=true diff --git a/.gitignore b/.gitignore index d3bb70515..8a01f6260 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,12 @@ Thumbs.db # Docker build artifacts *.tar + +# Veta agent memory +.veta/ + +# greger.el conversation +*.greger + +# playwright-cli +.playwright-cli/ \ No newline at end of file diff --git a/src/auth/middleware.test.ts b/src/auth/middleware.test.ts index 1c49ce605..caeb71061 100644 --- a/src/auth/middleware.test.ts +++ b/src/auth/middleware.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { isDevMode, extractJWT } from './middleware'; +import { isDevMode, isE2ETestMode, extractJWT } from './middleware'; import type { MoltbotEnv } from '../types'; import type { Context } from 'hono'; import type { AppEnv } from '../types'; @@ -32,6 +32,28 @@ describe('isDevMode', () => { }); }); +describe('isE2ETestMode', () => { + it('returns true when E2E_TEST_MODE is "true"', () => { + const env = createMockEnv({ E2E_TEST_MODE: 'true' }); + expect(isE2ETestMode(env)).toBe(true); + }); + + it('returns false when E2E_TEST_MODE is undefined', () => { + const env = createMockEnv(); + expect(isE2ETestMode(env)).toBe(false); + }); + + it('returns false when E2E_TEST_MODE is "false"', () => { + const env = createMockEnv({ E2E_TEST_MODE: 'false' }); + expect(isE2ETestMode(env)).toBe(false); + }); + + it('returns false when E2E_TEST_MODE is any other value', () => { + const env = createMockEnv({ E2E_TEST_MODE: 'yes' }); + expect(isE2ETestMode(env)).toBe(false); + }); +}); + describe('extractJWT', () => { // Helper to create a mock context function createMockContext(options: { @@ -158,6 +180,17 @@ describe('createAccessMiddleware', () => { expect(setMock).toHaveBeenCalledWith('accessUser', { email: 'dev@localhost', name: 'Dev User' }); }); + it('skips auth and sets dev user when E2E_TEST_MODE is true', async () => { + const { c, setMock } = createFullMockContext({ env: { E2E_TEST_MODE: 'true' } }); + const middleware = createAccessMiddleware({ type: 'json' }); + const next = vi.fn(); + + await middleware(c, next); + + expect(next).toHaveBeenCalled(); + expect(setMock).toHaveBeenCalledWith('accessUser', { email: 'dev@localhost', name: 'Dev User' }); + }); + it('returns 500 JSON error when CF Access not configured', async () => { const { c, jsonMock } = createFullMockContext({ env: {} }); const middleware = createAccessMiddleware({ type: 'json' }); diff --git a/src/auth/middleware.ts b/src/auth/middleware.ts index a1b7d2296..0b170a995 100644 --- a/src/auth/middleware.ts +++ b/src/auth/middleware.ts @@ -13,12 +13,19 @@ export interface AccessMiddlewareOptions { } /** - * Check if running in development mode (skips CF Access auth) + * Check if running in development mode (skips CF Access auth + device pairing) */ export function isDevMode(env: MoltbotEnv): boolean { return env.DEV_MODE === 'true'; } +/** + * Check if running in E2E test mode (skips CF Access auth but keeps device pairing) + */ +export function isE2ETestMode(env: MoltbotEnv): boolean { + return env.E2E_TEST_MODE === 'true'; +} + /** * Extract JWT from request headers or cookies */ @@ -42,8 +49,8 @@ export function createAccessMiddleware(options: AccessMiddlewareOptions) { const { type, redirectOnMissing = false } = options; return async (c: Context, next: Next) => { - // Skip auth in dev mode - if (isDevMode(c.env)) { + // Skip auth in dev mode or E2E test mode + if (isDevMode(c.env) || isE2ETestMode(c.env)) { c.set('accessUser', { email: 'dev@localhost', name: 'Dev User' }); return next(); } diff --git a/src/index.ts b/src/index.ts index 3ee1f5c20..6cf77e6ff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,11 +38,11 @@ function transformErrorMessage(message: string, host: string): string { if (message.includes('gateway token missing') || message.includes('gateway token mismatch')) { return `Invalid or missing token. Visit https://${host}?token={REPLACE_WITH_YOUR_TOKEN}`; } - + if (message.includes('pairing required')) { return `Pairing required. Visit https://${host}/_admin/`; } - + return message; } @@ -54,17 +54,21 @@ export { Sandbox }; */ function validateRequiredEnv(env: MoltbotEnv): string[] { const missing: string[] = []; + const isTestMode = env.DEV_MODE === 'true' || env.E2E_TEST_MODE === 'true'; if (!env.MOLTBOT_GATEWAY_TOKEN) { missing.push('MOLTBOT_GATEWAY_TOKEN'); } - if (!env.CF_ACCESS_TEAM_DOMAIN) { - missing.push('CF_ACCESS_TEAM_DOMAIN'); - } + // CF Access vars not required in dev/test mode since auth is skipped + if (!isTestMode) { + if (!env.CF_ACCESS_TEAM_DOMAIN) { + missing.push('CF_ACCESS_TEAM_DOMAIN'); + } - if (!env.CF_ACCESS_AUD) { - missing.push('CF_ACCESS_AUD'); + if (!env.CF_ACCESS_AUD) { + missing.push('CF_ACCESS_AUD'); + } } // Check for AI Gateway or direct Anthropic configuration @@ -94,12 +98,12 @@ function validateRequiredEnv(env: MoltbotEnv): string[] { */ function buildSandboxOptions(env: MoltbotEnv): SandboxOptions { const sleepAfter = env.SANDBOX_SLEEP_AFTER?.toLowerCase() || 'never'; - + // 'never' means keep the container alive indefinitely if (sleepAfter === 'never') { return { keepAlive: true }; } - + // Otherwise, use the specified duration return { sleepAfter }; } @@ -147,28 +151,28 @@ app.route('/cdp', cdp); // Middleware: Validate required environment variables (skip in dev mode and for debug routes) app.use('*', async (c, next) => { const url = new URL(c.req.url); - + // Skip validation for debug routes (they have their own enable check) if (url.pathname.startsWith('/debug')) { return next(); } - + // Skip validation in dev mode if (c.env.DEV_MODE === 'true') { return next(); } - + const missingVars = validateRequiredEnv(c.env); if (missingVars.length > 0) { console.error('[CONFIG] Missing required environment variables:', missingVars.join(', ')); - + const acceptsHtml = c.req.header('Accept')?.includes('text/html'); if (acceptsHtml) { // Return a user-friendly HTML error page const html = configErrorHtml.replace('{{MISSING_VARS}}', missingVars.join(', ')); return c.html(html, 503); } - + // Return JSON error for API requests return c.json({ error: 'Configuration error', @@ -177,7 +181,7 @@ app.use('*', async (c, next) => { hint: 'Set these using: wrangler secret put ', }, 503); } - + return next(); }); @@ -185,11 +189,11 @@ app.use('*', async (c, next) => { app.use('*', async (c, next) => { // Determine response type based on Accept header const acceptsHtml = c.req.header('Accept')?.includes('text/html'); - const middleware = createAccessMiddleware({ + const middleware = createAccessMiddleware({ type: acceptsHtml ? 'html' : 'json', - redirectOnMissing: acceptsHtml + redirectOnMissing: acceptsHtml }); - + return middleware(c, next); }); @@ -222,21 +226,21 @@ app.all('*', async (c) => { // Check if gateway is already running const existingProcess = await findExistingMoltbotProcess(sandbox); const isGatewayReady = existingProcess !== null && existingProcess.status === 'running'; - + // For browser requests (non-WebSocket, non-API), show loading page if gateway isn't ready const isWebSocketRequest = request.headers.get('Upgrade')?.toLowerCase() === 'websocket'; const acceptsHtml = request.headers.get('Accept')?.includes('text/html'); - + if (!isGatewayReady && !isWebSocketRequest && acceptsHtml) { console.log('[PROXY] Gateway not ready, serving loading page'); - + // Start the gateway in the background (don't await) c.executionCtx.waitUntil( ensureMoltbotGateway(sandbox, c.env).catch((err: Error) => { console.error('[PROXY] Background gateway start failed:', err); }) ); - + // Return the loading page immediately return c.html(loadingPageHtml); } @@ -267,31 +271,31 @@ app.all('*', async (c) => { console.log('[WS] Proxying WebSocket connection to Moltbot'); console.log('[WS] URL:', request.url); console.log('[WS] Search params:', url.search); - + // Get WebSocket connection to the container const containerResponse = await sandbox.wsConnect(request, MOLTBOT_PORT); console.log('[WS] wsConnect response status:', containerResponse.status); - + // Get the container-side WebSocket const containerWs = containerResponse.webSocket; if (!containerWs) { console.error('[WS] No WebSocket in container response - falling back to direct proxy'); return containerResponse; } - + console.log('[WS] Got container WebSocket, setting up interception'); - + // Create a WebSocket pair for the client const [clientWs, serverWs] = Object.values(new WebSocketPair()); - + // Accept both WebSockets serverWs.accept(); containerWs.accept(); - + console.log('[WS] Both WebSockets accepted'); console.log('[WS] containerWs.readyState:', containerWs.readyState); console.log('[WS] serverWs.readyState:', serverWs.readyState); - + // Relay messages from client to container serverWs.addEventListener('message', (event) => { console.log('[WS] Client -> Container:', typeof event.data, typeof event.data === 'string' ? event.data.slice(0, 200) : '(binary)'); @@ -301,12 +305,12 @@ app.all('*', async (c) => { console.log('[WS] Container not open, readyState:', containerWs.readyState); } }); - + // Relay messages from container to client, with error transformation containerWs.addEventListener('message', (event) => { console.log('[WS] Container -> Client (raw):', typeof event.data, typeof event.data === 'string' ? event.data.slice(0, 500) : '(binary)'); let data = event.data; - + // Try to intercept and transform error messages if (typeof data === 'string') { try { @@ -322,20 +326,20 @@ app.all('*', async (c) => { console.log('[WS] Not JSON or parse error:', e); } } - + if (serverWs.readyState === WebSocket.OPEN) { serverWs.send(data); } else { console.log('[WS] Server not open, readyState:', serverWs.readyState); } }); - + // Handle close events serverWs.addEventListener('close', (event) => { console.log('[WS] Client closed:', event.code, event.reason); containerWs.close(event.code, event.reason); }); - + containerWs.addEventListener('close', (event) => { console.log('[WS] Container closed:', event.code, event.reason); // Transform the close reason (truncate to 123 bytes max for WebSocket spec) @@ -346,18 +350,18 @@ app.all('*', async (c) => { console.log('[WS] Transformed close reason:', reason); serverWs.close(event.code, reason); }); - + // Handle errors serverWs.addEventListener('error', (event) => { console.error('[WS] Client error:', event); containerWs.close(1011, 'Client error'); }); - + containerWs.addEventListener('error', (event) => { console.error('[WS] Container error:', event); serverWs.close(1011, 'Container error'); }); - + console.log('[WS] Returning intercepted WebSocket response'); return new Response(null, { status: 101, @@ -368,12 +372,12 @@ app.all('*', async (c) => { console.log('[HTTP] Proxying:', url.pathname + url.search); const httpResponse = await sandbox.containerFetch(request, MOLTBOT_PORT); console.log('[HTTP] Response status:', httpResponse.status); - + // Add debug header to verify worker handled the request const newHeaders = new Headers(httpResponse.headers); newHeaders.set('X-Worker-Debug', 'proxy-to-moltbot'); newHeaders.set('X-Debug-Path', url.pathname); - + return new Response(httpResponse.body, { status: httpResponse.status, statusText: httpResponse.statusText, @@ -395,7 +399,7 @@ async function scheduled( console.log('[cron] Starting backup sync to R2...'); const result = await syncToR2(sandbox, env); - + if (result.success) { console.log('[cron] Backup sync completed successfully at', result.lastSync); } else { diff --git a/src/types.ts b/src/types.ts index bb82c8ca4..6287bc708 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,6 +18,7 @@ export interface MoltbotEnv { CLAWDBOT_BIND_MODE?: string; DEV_MODE?: string; // Set to 'true' for local dev (skips CF Access auth + moltbot device pairing) + E2E_TEST_MODE?: string; // Set to 'true' for E2E tests (skips CF Access auth but keeps device pairing) DEBUG_ROUTES?: string; // Set to 'true' to enable /debug/* routes SANDBOX_SLEEP_AFTER?: string; // How long before sandbox sleeps: 'never' (default), or duration like '10m', '1h' TELEGRAM_BOT_TOKEN?: string; diff --git a/test/e2e/_setup.txt b/test/e2e/_setup.txt new file mode 100644 index 000000000..fe8350b0f --- /dev/null +++ b/test/e2e/_setup.txt @@ -0,0 +1,26 @@ +=== +start moltworker server +=== +./start-server -v +--- +{{ s }} +--- +where +* strip(s) endswith "ready" + +=== +start playwright browser +=== +./start-browser +--- +ready + +=== +start video recording +=== +./pw --session=moltworker-e2e video-start +--- +{{ output }} +--- +where +* output contains "Video recording started" diff --git a/test/e2e/_teardown.txt b/test/e2e/_teardown.txt new file mode 100644 index 000000000..575c417a7 --- /dev/null +++ b/test/e2e/_teardown.txt @@ -0,0 +1,46 @@ +=== +stop video recording +=== +./pw --session=moltworker-e2e video-stop +--- +{{ output }} +--- +where +* output contains "Video" + +=== +save video recording +=== +mkdir -p /tmp/moltworker-e2e-videos +datetime=$(date +%Y%m%d-%H%M%S) +for f in ./.playwright-cli/*.webm; do + if [ -f "$f" ]; then + cp "$f" "/tmp/moltworker-e2e-videos/${datetime}.webm" + echo "video saved to /tmp/moltworker-e2e-videos/${datetime}.webm" + fi +done +--- +{{ output }} +--- +where +* output contains "video saved to" + +=== +stop playwright browser +=== +./stop-browser +--- +{{ output }} +--- +where +* output contains "stopped" + +=== +stop moltworker server +=== +./stop-server +--- +{{ s }} +--- +where +* strip(s) endswith "stopped" diff --git a/test/e2e/fixture/pw b/test/e2e/fixture/pw new file mode 100755 index 000000000..f4472c9f8 --- /dev/null +++ b/test/e2e/fixture/pw @@ -0,0 +1,28 @@ +#!/bin/bash +# Wrapper for playwright-cli that returns non-zero exit code on errors. +# +# playwright-cli has a bug where it ignores the isError flag returned from +# the daemon. In program.js line ~279, it only does: +# +# console.log(result.text); +# session.close(); +# +# But it should also do: +# +# if (result.isError) process.exit(1); +# +# Until this is fixed upstream, we detect errors by checking for "### Error" +# in the output (which is the format used by browserServerBackend.js). +# +# See: https://github.com/microsoft/playwright/blob/main/packages/playwright/src/mcp/terminal/program.ts + +output=$(playwright-cli "$@" 2>&1) +exit_code=$? + +echo "$output" + +if echo "$output" | grep -q "^### Error"; then + exit 1 +fi + +exit $exit_code diff --git a/test/e2e/fixture/start-browser b/test/e2e/fixture/start-browser new file mode 100755 index 000000000..c8887f655 --- /dev/null +++ b/test/e2e/fixture/start-browser @@ -0,0 +1,27 @@ +#!/bin/bash +# Start playwright-cli browser session for E2E testing + +set -e + +SESSION_NAME="moltworker-e2e" + +# Stop and delete any existing session (delete needed to change headed/headless mode) +playwright-cli session-stop "$SESSION_NAME" >/dev/null 2>&1 || true +playwright-cli session-delete "$SESSION_NAME" >/dev/null 2>&1 || true + +# Build the open command args +GLOBAL_ARGS=("--session=$SESSION_NAME") + +# Run headed if PLAYWRIGHT_HEADED is set +if [ "${PLAYWRIGHT_HEADED:-}" = "1" ] || [ "${PLAYWRIGHT_HEADED:-}" = "true" ]; then + GLOBAL_ARGS+=("--headed") +fi + +# Open the browser to a blank page first (will navigate later in tests) +# Redirect all playwright output to /dev/null since it's very verbose +playwright-cli "${GLOBAL_ARGS[@]}" open "about:blank" >/dev/null 2>&1 & + +# Give it a moment to start +sleep 2 + +echo "ready" diff --git a/test/e2e/fixture/start-server b/test/e2e/fixture/start-server new file mode 100755 index 000000000..9dd83e693 --- /dev/null +++ b/test/e2e/fixture/start-server @@ -0,0 +1,142 @@ +#!/bin/bash +# Start the moltworker for E2E testing + +set -e + +VERBOSE=false +if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then + VERBOSE=true +fi + +log() { + if [ "$VERBOSE" = true ]; then + echo "[start-server] $*" >&2 + fi +} + +# Support running directly (not via cctr) for manual debugging +if [ -z "$CCTR_TEST_PATH" ]; then + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + CCTR_TEST_PATH="$(dirname "$SCRIPT_DIR")" + log "CCTR_TEST_PATH not set, using: $CCTR_TEST_PATH" +fi +if [ -z "$CCTR_FIXTURE_DIR" ]; then + CCTR_FIXTURE_DIR="/tmp/e2e-manual" + mkdir -p "$CCTR_FIXTURE_DIR" + log "CCTR_FIXTURE_DIR not set, using: $CCTR_FIXTURE_DIR" +fi + +PROJECT_DIR="$(cd "$CCTR_TEST_PATH/../.." && pwd)" +PORT=8686 +GATEWAY_TOKEN="e2e-test-token-1234567890" + +log "Project directory: $PROJECT_DIR" +log "Fixture directory: $CCTR_FIXTURE_DIR" +log "Port: $PORT" +log "Gateway token: $GATEWAY_TOKEN" + +# Kill any existing server on our port +log "Killing any existing server on port $PORT..." +pkill -f "wrangler.*--port.*$PORT" 2>/dev/null || true +pkill -f "wrangler dev" 2>/dev/null || true +sleep 0.5 + +# Stop any existing sandbox containers +log "Stopping any existing sandbox containers..." +docker ps -q --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker stop 2>/dev/null || true +docker ps -aq --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker rm 2>/dev/null || true + +cd "$PROJECT_DIR" + +# Install dependencies if needed +if [ ! -d node_modules ]; then + log "Installing dependencies..." + npm install --silent 2>/dev/null +fi + +# Build the project (required after code changes) +log "Building project..." +if [ "$VERBOSE" = true ]; then + npm run build >&2 +else + npm run build >/dev/null 2>&1 +fi + +# Write token to a file so tests can read it +echo "$GATEWAY_TOKEN" > "$CCTR_FIXTURE_DIR/gateway-token.txt" + +# Generate complete .dev.vars.e2e by copying from .dev.vars and overriding what we need +log "Creating .dev.vars.e2e..." +cat > "$CCTR_FIXTURE_DIR/.dev.vars.e2e" << EOF +E2E_TEST_MODE=true +DEBUG_ROUTES=true +MOLTBOT_GATEWAY_TOKEN=$GATEWAY_TOKEN +EOF + +# Copy all other settings from existing .dev.vars (except the ones we override) +if [ -f "$PROJECT_DIR/.dev.vars" ]; then + log "Copying settings from .dev.vars..." + grep -v -E "^(E2E_TEST_MODE|DEV_MODE|DEBUG_ROUTES|MOLTBOT_GATEWAY_TOKEN)=" "$PROJECT_DIR/.dev.vars" >> "$CCTR_FIXTURE_DIR/.dev.vars.e2e" 2>/dev/null || true +fi + +if [ "$VERBOSE" = true ]; then + log "Generated .dev.vars.e2e contents:" + cat "$CCTR_FIXTURE_DIR/.dev.vars.e2e" >&2 +fi + +# Temporarily rename .dev.vars so wrangler ONLY reads our test config +if [ -f "$PROJECT_DIR/.dev.vars" ]; then + log "Temporarily moving .dev.vars out of the way..." + mv "$PROJECT_DIR/.dev.vars" "$PROJECT_DIR/.dev.vars.e2e-backup" +fi + +# Copy our test config to .dev.vars location so wrangler finds it +cp "$CCTR_FIXTURE_DIR/.dev.vars.e2e" "$PROJECT_DIR/.dev.vars" + +log "Starting wrangler dev..." +# Start wrangler in background, logging to file +# Use nohup and redirect all output to detach from terminal +nohup npx wrangler dev \ + --port "$PORT" \ + > "$CCTR_FIXTURE_DIR/wrangler.log" 2>&1 & +WRANGLER_PID=$! +echo $WRANGLER_PID > "$CCTR_FIXTURE_DIR/wrangler.pid" +log "Wrangler PID: $WRANGLER_PID" + +# In verbose mode, tail the log in background so we can see output +if [ "$VERBOSE" = true ]; then + tail -f "$CCTR_FIXTURE_DIR/wrangler.log" >&2 & + TAIL_PID=$! +fi + +# Give wrangler a moment to read the config, then restore original .dev.vars +sleep 2 +if [ -f "$PROJECT_DIR/.dev.vars.e2e-backup" ]; then + log "Restoring original .dev.vars..." + mv "$PROJECT_DIR/.dev.vars.e2e-backup" "$PROJECT_DIR/.dev.vars" +fi + +# Wait for server to be ready (container startup can take 1-2 minutes) +log "Waiting for server to be ready..." +for i in {1..180}; do + # Check for 200 response, not just any response + status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$PORT/?token=$GATEWAY_TOKEN" 2>/dev/null || echo "000") + if [ "$status" = "200" ]; then + log "Server is ready! (HTTP $status)" + log "Open: http://localhost:$PORT/?token=$GATEWAY_TOKEN" + # Kill the tail process if running + [ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true + echo "ready" + exit 0 + fi + if [ "$VERBOSE" = true ] && [ $((i % 10)) -eq 0 ]; then + log "Still waiting... ($i seconds, last status: $status)" + fi + sleep 1 +done + +log "Timeout waiting for server" +# Kill the tail process if running +[ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true +cat "$CCTR_FIXTURE_DIR/wrangler.log" >&2 +exit 1 diff --git a/test/e2e/fixture/stop-browser b/test/e2e/fixture/stop-browser new file mode 100755 index 000000000..e1e4a5ae0 --- /dev/null +++ b/test/e2e/fixture/stop-browser @@ -0,0 +1,8 @@ +#!/bin/bash +# Stop playwright-cli browser session + +SESSION_NAME="moltworker-e2e" + +playwright-cli session-stop "$SESSION_NAME" 2>/dev/null || true + +echo "stopped" diff --git a/test/e2e/fixture/stop-server b/test/e2e/fixture/stop-server new file mode 100755 index 000000000..82fb2d61d --- /dev/null +++ b/test/e2e/fixture/stop-server @@ -0,0 +1,37 @@ +#!/bin/bash +# Stop the moltworker and clean up + +set -e + +# Stop wrangler if running +if [ -f "$CCTR_FIXTURE_DIR/wrangler.pid" ]; then + pid=$(cat "$CCTR_FIXTURE_DIR/wrangler.pid") + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + # Wait for it to die + for i in {1..10}; do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.5 + done + # Force kill if still running + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$CCTR_FIXTURE_DIR/wrangler.pid" +fi + +# Kill any remaining wrangler processes on our port +pkill -f "wrangler.*--port.*8686" 2>/dev/null || true +pkill -f "wrangler dev" 2>/dev/null || true + +# Stop and remove sandbox containers +docker ps -q --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker stop 2>/dev/null || true +docker ps -aq --filter "name=workerd-moltbot-sandbox" 2>/dev/null | xargs -r docker rm 2>/dev/null || true + +# Clean up temp files +rm -f "$CCTR_FIXTURE_DIR/.dev.vars.e2e" +rm -f "$CCTR_FIXTURE_DIR/wrangler.log" +rm -f "$CCTR_FIXTURE_DIR/gateway-token.txt" + +echo "stopped" diff --git a/test/e2e/pairing_and_conversation.txt b/test/e2e/pairing_and_conversation.txt new file mode 100644 index 000000000..86717189a --- /dev/null +++ b/test/e2e/pairing_and_conversation.txt @@ -0,0 +1,97 @@ +=== +navigate to main page to trigger pairing request +%require +=== +TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") +./pw --session=moltworker-e2e open "http://localhost:8686/?token=$TOKEN" +--- + +=== +wait for websocket connection to establish +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForLoadState('networkidle'); +}" +--- + +=== +navigate to admin page to approve device +%require +=== +TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") +./pw --session=moltworker-e2e open "http://localhost:8686/_admin/?token=$TOKEN" +--- + +=== +wait for pending devices section to load +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=Pending Pairing Requests', { timeout: 60000 }); +}" +--- + +=== +wait for Approve All button and click it +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + const btn = await page.waitForSelector('button:has-text(\"Approve All\")', { timeout: 60000 }); + await btn.click(); +}" +--- + +=== +wait for approval to complete +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=No pending pairing requests', { timeout: 60000 }); +}" +--- + +=== +navigate back to main chat page +%require +=== +TOKEN=$(cat "$CCTR_FIXTURE_DIR/gateway-token.txt") +./pw --session=moltworker-e2e open "http://localhost:8686/?token=$TOKEN" +--- + +=== +wait for chat interface to load +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('textarea', { timeout: 60000 }); +}" +--- + +=== +type math question into chat +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + const textarea = await page.waitForSelector('textarea'); + await textarea.fill('What is 847293 + 651824? Reply with just the number.'); +}" +--- + +=== +click send button +%require +=== +./pw --session=moltworker-e2e run-code "async page => { + const btn = await page.waitForSelector('button:has-text(\"Send\")'); + await btn.click(); +}" +--- + +=== +wait for response containing the correct answer +=== +./pw --session=moltworker-e2e run-code "async page => { + await page.waitForSelector('text=1499117', { timeout: 120000 }); +}" +--- From 31d9d4c76f21422ac21a27534962ba46a83b27bd Mon Sep 17 00:00:00 2001 From: Andreas Jansson Date: Mon, 2 Feb 2026 11:29:44 +0100 Subject: [PATCH 2/2] Add GitHub Actions workflow for E2E tests with video recording - Add e2e job to test.yml (runs in parallel with unit tests) - Convert webm to mp4 using ffmpeg - Generate thumbnail with play button overlay using ImageMagick - Upload video and thumbnail to e2e-artifacts branch - Post PR comment with clickable thumbnail linking to video --- .github/workflows/test.yml | 125 +++++++++++++++++++++++++++++++++- test/e2e/fixture/start-server | 9 +++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9012e6c11..8eea450a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,9 +5,10 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: jobs: - test: + unit: runs-on: ubuntu-latest steps: @@ -27,3 +28,125 @@ jobs: - name: Run tests run: npm test + + e2e: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Playwright + run: npx playwright install --with-deps chromium + + - name: Install playwright-cli + run: npm install -g @playwright/cli + + - name: Install cctr + uses: taiki-e/install-action@v2 + with: + tool: cctr + + - name: Run E2E tests + id: e2e + continue-on-error: true + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + AI_GATEWAY_BASE_URL: ${{ secrets.AI_GATEWAY_BASE_URL }} + run: cctr -vv test/e2e + + - name: Convert video and generate thumbnail + id: convert + if: always() + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg imagemagick bc + if ls /tmp/moltworker-e2e-videos/*.webm 1>/dev/null 2>&1; then + for webm in /tmp/moltworker-e2e-videos/*.webm; do + mp4="${webm%.webm}.mp4" + thumb="${webm%.webm}.png" + + # Convert to mp4 + ffmpeg -y -i "$webm" -c:v libx264 -preset fast -crf 22 -c:a aac "$mp4" + + # Extract middle frame as thumbnail + duration=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$mp4") + midpoint=$(echo "$duration / 2" | bc -l) + ffmpeg -y -ss "$midpoint" -i "$mp4" -vframes 1 -update 1 -q:v 2 "$thumb" + + # Add play button overlay using ImageMagick + width=$(identify -format '%w' "$thumb") + height=$(identify -format '%h' "$thumb") + cx=$((width / 2)) + cy=$((height / 2)) + convert "$thumb" \ + -fill 'rgba(0,0,0,0.6)' -draw "circle ${cx},${cy} $((cx+50)),${cy}" \ + -fill 'white' -draw "polygon $((cx-15)),$((cy-25)) $((cx-15)),$((cy+25)) $((cx+30)),${cy}" \ + "$thumb" + + echo "video_path=$mp4" >> $GITHUB_OUTPUT + echo "video_name=$(basename $mp4)" >> $GITHUB_OUTPUT + echo "thumb_path=$thumb" >> $GITHUB_OUTPUT + echo "thumb_name=$(basename $thumb)" >> $GITHUB_OUTPUT + done + echo "has_video=true" >> $GITHUB_OUTPUT + else + echo "has_video=false" >> $GITHUB_OUTPUT + fi + + - name: Prepare video for upload + id: prepare + if: always() && steps.convert.outputs.has_video == 'true' + run: | + mkdir -p /tmp/e2e-video-upload/videos/${{ github.run_id }} + cp "${{ steps.convert.outputs.video_path }}" /tmp/e2e-video-upload/videos/${{ github.run_id }}/ + cp "${{ steps.convert.outputs.thumb_path }}" /tmp/e2e-video-upload/videos/${{ github.run_id }}/ + echo "video_url=https://github.com/${{ github.repository }}/raw/e2e-artifacts/videos/${{ github.run_id }}/${{ steps.convert.outputs.video_name }}" >> $GITHUB_OUTPUT + echo "thumb_url=https://github.com/${{ github.repository }}/raw/e2e-artifacts/videos/${{ github.run_id }}/${{ steps.convert.outputs.thumb_name }}" >> $GITHUB_OUTPUT + + - name: Upload video to e2e-artifacts branch + if: always() && steps.convert.outputs.has_video == 'true' + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: /tmp/e2e-video-upload + publish_branch: e2e-artifacts + keep_files: true + + - name: Comment on PR with video + if: always() && github.event_name == 'pull_request' && steps.prepare.outputs.video_url + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## E2E Test Recording + + ${{ steps.e2e.outcome == 'success' && '✅ Tests passed' || '❌ Tests failed' }} + + [![E2E Test Video](${{ steps.prepare.outputs.thumb_url }})](${{ steps.prepare.outputs.video_url }}) + + - name: Add video link to summary + if: always() + run: | + echo "## E2E Test Recording" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.convert.outputs.has_video }}" == "true" ]; then + echo "📹 [Download video](${{ steps.prepare.outputs.video_url }})" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ No video recording found" >> $GITHUB_STEP_SUMMARY + fi + + - name: Fail if E2E tests failed + if: steps.e2e.outcome == 'failure' + run: exit 1 diff --git a/test/e2e/fixture/start-server b/test/e2e/fixture/start-server index 9dd83e693..8a230272a 100755 --- a/test/e2e/fixture/start-server +++ b/test/e2e/fixture/start-server @@ -79,6 +79,13 @@ if [ -f "$PROJECT_DIR/.dev.vars" ]; then grep -v -E "^(E2E_TEST_MODE|DEV_MODE|DEBUG_ROUTES|MOLTBOT_GATEWAY_TOKEN)=" "$PROJECT_DIR/.dev.vars" >> "$CCTR_FIXTURE_DIR/.dev.vars.e2e" 2>/dev/null || true fi +# Also pick up API keys from environment (for CI) +for var in AI_GATEWAY_API_KEY AI_GATEWAY_BASE_URL ANTHROPIC_API_KEY OPENAI_API_KEY; do + if [ -n "${!var}" ]; then + echo "$var=${!var}" >> "$CCTR_FIXTURE_DIR/.dev.vars.e2e" + fi +done + if [ "$VERBOSE" = true ]; then log "Generated .dev.vars.e2e contents:" cat "$CCTR_FIXTURE_DIR/.dev.vars.e2e" >&2 @@ -126,6 +133,8 @@ for i in {1..180}; do log "Open: http://localhost:$PORT/?token=$GATEWAY_TOKEN" # Kill the tail process if running [ -n "$TAIL_PID" ] && kill $TAIL_PID 2>/dev/null || true + # Small delay to let stderr flush before stdout + sleep 0.1 echo "ready" exit 0 fi