From 5dd791ebd0372a2d0090566abba1dd2e21a0d1bf Mon Sep 17 00:00:00 2001 From: Jim Carter III Date: Mon, 9 Feb 2026 18:33:35 -0800 Subject: [PATCH] fast-qa: SSE streaming execution + bug fixes - Add /api/execute-test endpoint with SSE streaming to avoid Vercel timeout issues - Client-side parallelism (3 concurrent) with real-time step progress - Fix gpt-5-nano (reasoning model) returning empty content - switch to gpt-4.1-nano - Fix type error in generate-report route (extractedData parsing) - Fix frontend data key mismatch (testCases vs tests) - Mark old /api/execute-tests as deprecated The original /api/execute-tests blocks until all tests complete, causing 504 Gateway Timeout on Vercel free tier (30s edge limit). The new /api/execute-test (singular) streams Mino SSE events to the client, keeping the connection alive and providing real-time progress updates. --- fast-qa/app/api/execute-test/route.ts | 297 ++++++++++++++++++++ fast-qa/app/api/execute-tests/route.ts | 5 + fast-qa/app/api/generate-report/route.ts | 2 +- fast-qa/app/api/generate-tests/route.ts | 2 +- fast-qa/lib/hooks.ts | 330 +++++++++++++++-------- 5 files changed, 520 insertions(+), 116 deletions(-) create mode 100644 fast-qa/app/api/execute-test/route.ts diff --git a/fast-qa/app/api/execute-test/route.ts b/fast-qa/app/api/execute-test/route.ts new file mode 100644 index 0000000..612b4ed --- /dev/null +++ b/fast-qa/app/api/execute-test/route.ts @@ -0,0 +1,297 @@ +/** + * POST /api/execute-test - Execute a SINGLE test case against Mino via SSE streaming + * + * This endpoint now returns a streaming SSE response that proxies Mino's events + * to the client in real-time. This avoids Vercel timeout limits since streaming + * responses don't count against execution time once the first byte is sent. + * + * Uses Edge Runtime for compatibility with ReadableStream. + */ + +export const runtime = 'edge'; + +import type { TestCase, TestResult, QASettings } from '@/types'; +import { generateId, parseSSELine, isCompleteEvent, isErrorEvent, formatStepMessage } from '@/lib/utils'; +import { generateTestResultSummary } from '@/lib/ai-client'; + +interface ExecuteTestRequest { + testCase: TestCase; + websiteUrl: string; + settings?: Partial; +} + +const MINO_API_URL = "https://mino.ai/v1/automation/run-sse"; + +export async function POST(request: Request) { + try { + const body: ExecuteTestRequest = await request.json(); + const { testCase, websiteUrl, settings } = body; + + if (!testCase) { + return Response.json({ error: 'No test case provided' }, { status: 400 }); + } + + if (!websiteUrl) { + return Response.json({ error: 'No website URL provided' }, { status: 400 }); + } + + const apiKey = process.env.MINO_API_KEY; + if (!apiKey) { + return Response.json({ error: 'MINO_API_KEY not configured' }, { status: 500 }); + } + + // Build the goal from test case + let goal = testCase.description; + if (testCase.expectedOutcome) { + goal += `\n\nExpected outcome: ${testCase.expectedOutcome}`; + goal += `\n\nAfter completing the steps, verify that the expected outcome is met. Return a JSON object with { "success": true/false, "reason": "explanation" }`; + } + + const minoConfig = { + url: websiteUrl, + goal, + browser_profile: settings?.browserProfile || 'lite', + proxy_config: settings?.proxyEnabled + ? { + enabled: true, + country_code: settings.proxyCountry || 'US', + } + : undefined, + }; + + // Create a readable stream that proxies Mino's SSE events + const stream = new ReadableStream({ + async start(controller) { + const startTime = Date.now(); + const collectedSteps: string[] = []; + let streamingUrl: string | undefined; + + try { + // Start the Mino automation request + const minoResponse = await fetch(MINO_API_URL, { + method: 'POST', + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(minoConfig), + }); + + if (!minoResponse.ok) { + const errorText = await minoResponse.text(); + throw new Error(`Mino API request failed: ${minoResponse.status} ${errorText}`); + } + + if (!minoResponse.body) { + throw new Error('Mino response body is null'); + } + + // Send initial event to client + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'test_start', + testCaseId: testCase.id, + timestamp: Date.now(), + })}\n\n`)); + + const reader = minoResponse.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + + for (const line of lines) { + const event = parseSSELine(line); + if (!event) continue; + + // Capture streaming URL + if (event.streamingUrl && !streamingUrl) { + streamingUrl = event.streamingUrl; + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'streaming_url', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { streamingUrl: event.streamingUrl }, + })}\n\n`)); + } + + // Handle step events + if (event.type === 'STEP') { + const stepMessage = formatStepMessage(event); + collectedSteps.push(stepMessage); + + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'step_progress', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { + stepDescription: stepMessage, + currentStep: collectedSteps.length, + }, + })}\n\n`)); + } + + // Check for completion + if (isCompleteEvent(event)) { + const completedAt = Date.now(); + const duration = completedAt - startTime; + + // Determine success from Mino response + let success = true; + let error: string | undefined; + let reason: string | undefined; + let extractedData: Record | undefined; + + if (event.resultJson && typeof event.resultJson === 'object') { + const result = event.resultJson as Record; + if ('success' in result) success = Boolean(result.success); + if ('error' in result && typeof result.error === 'string') error = result.error; + if ('reason' in result && typeof result.reason === 'string') reason = result.reason; + if ('extractedData' in result) extractedData = result.extractedData as Record; + } + + // Generate AI summary if needed + if (!reason || reason === error) { + try { + reason = await generateTestResultSummary( + { + title: testCase.title, + description: testCase.description, + expectedOutcome: testCase.expectedOutcome, + }, + { + status: success ? 'passed' : 'failed', + steps: collectedSteps, + error, + duration, + }, + websiteUrl + ); + } catch (summaryError) { + console.error('Failed to generate AI summary:', summaryError); + } + } + + const testResult: TestResult = { + id: generateId(), + testCaseId: testCase.id, + status: success ? 'passed' : 'failed', + startedAt: startTime, + completedAt, + duration, + streamingUrl, + error, + reason, + steps: collectedSteps.length > 0 ? collectedSteps : undefined, + extractedData, + }; + + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'test_complete', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { result: testResult }, + })}\n\n`)); + + controller.close(); + return; + } + + // Check for errors + if (isErrorEvent(event)) { + const completedAt = Date.now(); + const duration = completedAt - startTime; + const errorMsg = event.message || 'Automation failed'; + + const testResult: TestResult = { + id: generateId(), + testCaseId: testCase.id, + status: 'failed', + startedAt: startTime, + completedAt, + duration, + streamingUrl, + error: errorMsg, + reason: errorMsg, + steps: collectedSteps.length > 0 ? collectedSteps : undefined, + }; + + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'test_error', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { result: testResult, error: errorMsg }, + })}\n\n`)); + + controller.close(); + return; + } + } + } + + // If we reach here without completion, it's an unexpected end + const testResult: TestResult = { + id: generateId(), + testCaseId: testCase.id, + status: 'error', + startedAt: startTime, + completedAt: Date.now(), + duration: Date.now() - startTime, + streamingUrl, + error: 'Stream ended without completion event', + steps: collectedSteps.length > 0 ? collectedSteps : undefined, + }; + + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'test_error', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { result: testResult, error: 'Stream ended unexpectedly' }, + })}\n\n`)); + + controller.close(); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + + const testResult: TestResult = { + id: generateId(), + testCaseId: testCase.id, + status: 'error', + startedAt: startTime, + completedAt: Date.now(), + duration: Date.now() - startTime, + streamingUrl, + error: errorMsg, + steps: collectedSteps.length > 0 ? collectedSteps : undefined, + }; + + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ + type: 'test_error', + testCaseId: testCase.id, + timestamp: Date.now(), + data: { result: testResult, error: errorMsg }, + })}\n\n`)); + + controller.close(); + } + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }, + }); + } catch (err) { + console.error('Error in execute-test API:', err); + const errorMessage = err instanceof Error ? err.message : 'Unknown error'; + return Response.json({ error: errorMessage }, { status: 500 }); + } +} diff --git a/fast-qa/app/api/execute-tests/route.ts b/fast-qa/app/api/execute-tests/route.ts index d593da7..4df15f4 100644 --- a/fast-qa/app/api/execute-tests/route.ts +++ b/fast-qa/app/api/execute-tests/route.ts @@ -1,3 +1,8 @@ +/** + * @deprecated Use /api/execute-test (singular) instead. + * This endpoint blocks until all tests complete, which causes 504 timeouts + * on Vercel's free tier. The new endpoint uses SSE streaming. + */ import { NextRequest } from 'next/server'; import { runMinoAutomation } from '@/lib/mino-client'; import { generateTestResultSummary } from '@/lib/ai-client'; diff --git a/fast-qa/app/api/generate-report/route.ts b/fast-qa/app/api/generate-report/route.ts index 5148e47..5b2397b 100644 --- a/fast-qa/app/api/generate-report/route.ts +++ b/fast-qa/app/api/generate-report/route.ts @@ -45,7 +45,7 @@ export async function POST(request: NextRequest) { }, { error: failedTest.error, - extractedData: sanitizePII(JSON.stringify(failedTest.extractedData)), + extractedData: JSON.parse(sanitizePII(JSON.stringify(failedTest.extractedData)) || '{}') as Record, }, projectUrl ); diff --git a/fast-qa/app/api/generate-tests/route.ts b/fast-qa/app/api/generate-tests/route.ts index 552f88e..8ab5f17 100644 --- a/fast-qa/app/api/generate-tests/route.ts +++ b/fast-qa/app/api/generate-tests/route.ts @@ -45,7 +45,7 @@ export async function POST(request: NextRequest) { } const openrouter = createOpenRouterProvider(); - const model = openrouter.chatModel('openai/gpt-5-nano'); + const model = openrouter.chatModel('openai/gpt-4.1-nano'); const system = `You are a QA test automation expert. Your job is to analyze raw text (which may include feature descriptions, user stories, requirements, or test scenarios) and generate a comprehensive list of test cases. Return your response as JSON. diff --git a/fast-qa/lib/hooks.ts b/fast-qa/lib/hooks.ts index 5973a2b..94af97a 100644 --- a/fast-qa/lib/hooks.ts +++ b/fast-qa/lib/hooks.ts @@ -1,7 +1,7 @@ "use client"; import { useState, useEffect, useCallback, useRef } from 'react'; -import type { TestCase, TestResult, TestEvent } from '@/types'; +import type { TestCase, TestResult } from '@/types'; /** * Hook for localStorage with SSR support @@ -41,78 +41,20 @@ export function useLocalStorage(key: string, initialValue: T): [T, (value: T } /** - * Hook for managing test execution with SSE + * Hook for managing test execution via SSE streaming. + * + * Each test case is executed individually via POST /api/execute-test which returns + * a streaming SSE response. The client manages parallelism (default 3 concurrent) + * and updates results in real-time as events arrive. */ export function useTestExecution(onComplete?: (finalResults: Map) => void) { const [isExecuting, setIsExecuting] = useState(false); const [results, setResults] = useState>(new Map()); const [error, setError] = useState(null); - const abortControllerRef = useRef(null); + const abortControllersRef = useRef([]); + const cancelledRef = useRef(false); const resultsRef = useRef>(new Map()); - - // Define handleTestEvent before executeTests so it can be used as a dependency - const handleTestEvent = useCallback((event: TestEvent) => { - const { testCaseId, data } = event; - - setResults((prev) => { - const newResults = new Map(prev); - const existing = newResults.get(testCaseId) || { - id: `result-${testCaseId}`, - testCaseId, - status: 'running' as const, - startedAt: Date.now(), - }; - - switch (event.type) { - case 'test_start': - newResults.set(testCaseId, { - ...existing, - status: 'running' as const, - startedAt: event.timestamp, - }); - break; - - case 'streaming_url': - newResults.set(testCaseId, { - ...existing, - streamingUrl: data?.streamingUrl, - }); - break; - - case 'step_progress': - newResults.set(testCaseId, { - ...existing, - currentStep: data?.currentStep, - totalSteps: data?.totalSteps, - currentStepDescription: data?.stepDescription, - }); - break; - - case 'test_complete': - if (data?.result) { - newResults.set(testCaseId, data.result); - // Also update the ref for immediate access - resultsRef.current.set(testCaseId, data.result); - } - break; - - case 'test_error': { - const errorResult = { - ...existing, - status: 'error' as const, - error: data?.error, - completedAt: event.timestamp, - }; - newResults.set(testCaseId, errorResult); - // Also update the ref for immediate access - resultsRef.current.set(testCaseId, errorResult); - break; - } - } - - return newResults; - }); - }, []); + const eventSourcesRef = useRef([]); const executeTests = useCallback(async ( testCases: TestCase[], @@ -125,71 +67,231 @@ export function useTestExecution(onComplete?: (finalResults: Map { + const initial = new Map(); + for (const tc of testCases) { + initial.set(tc.id, { + id: `result-${tc.id}`, + testCaseId: tc.id, + status: 'pending', + startedAt: Date.now(), + }); + } + return initial; + }); - abortControllerRef.current = new AbortController(); + const executeSingle = async (testCase: TestCase) => { + if (cancelledRef.current) return; - try { - const response = await fetch('/api/execute-tests', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - testCases, - websiteUrl, - parallelLimit, - }), - signal: abortControllerRef.current.signal, + const controller = new AbortController(); + abortControllersRef.current.push(controller); + + // Mark as running when starting + setResults((prev) => { + const next = new Map(prev); + next.set(testCase.id, { + id: `result-${testCase.id}`, + testCaseId: testCase.id, + status: 'running', + startedAt: Date.now(), + }); + return next; }); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } + return new Promise((resolve, reject) => { + // Start the SSE stream + fetch('/api/execute-test', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ testCase, websiteUrl }), + signal: controller.signal, + }) + .then(async (response) => { + if (!response.ok) { + const errBody = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); + throw new Error(errBody.error || `HTTP error ${response.status}`); + } - const reader = response.body?.getReader(); - if (!reader) { - throw new Error('No response body'); - } + if (!response.body) { + throw new Error('Response body is null'); + } - const decoder = new TextDecoder(); - let buffer = ''; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + if (cancelledRef.current) { + reader.cancel(); + resolve(); + return; + } + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + + try { + const eventData = JSON.parse(line.slice(6)); + + switch (eventData.type) { + case 'test_start': + // Already marked as running above + break; + + case 'streaming_url': + // Update with streaming URL + setResults((prev) => { + const next = new Map(prev); + const existing = next.get(testCase.id); + if (existing) { + next.set(testCase.id, { + ...existing, + streamingUrl: eventData.data.streamingUrl, + }); + } + return next; + }); + break; + + case 'step_progress': + // Update with step progress + setResults((prev) => { + const next = new Map(prev); + const existing = next.get(testCase.id); + if (existing) { + next.set(testCase.id, { + ...existing, + currentStepDescription: eventData.data.stepDescription, + currentStep: eventData.data.currentStep, + }); + } + return next; + }); + break; + + case 'test_complete': + // Final result + const finalResult = eventData.data.result; + setResults((prev) => { + const next = new Map(prev); + next.set(testCase.id, finalResult); + return next; + }); + resultsRef.current.set(testCase.id, finalResult); + resolve(); + return; + + case 'test_error': + // Error result + const errorResult = eventData.data.result; + setResults((prev) => { + const next = new Map(prev); + next.set(testCase.id, errorResult); + return next; + }); + resultsRef.current.set(testCase.id, errorResult); + resolve(); + return; + } + } catch (parseError) { + console.error('Failed to parse SSE event:', parseError); + } + } + } - while (true) { - const { done, value } = await reader.read(); - if (done) break; + // Stream ended without completion + resolve(); + } catch (streamError) { + reject(streamError); + } + }) + .catch((fetchError) => { + if (fetchError instanceof Error && fetchError.name === 'AbortError') { + resolve(); + return; + } + + const errorMessage = fetchError instanceof Error ? fetchError.message : 'Unknown error'; + const errorResult: TestResult = { + id: `result-${testCase.id}`, + testCaseId: testCase.id, + status: 'error', + startedAt: Date.now(), + completedAt: Date.now(), + error: errorMessage, + }; + + setResults((prev) => { + const next = new Map(prev); + next.set(testCase.id, errorResult); + return next; + }); + resultsRef.current.set(testCase.id, errorResult); + reject(fetchError); + }); + }); + }; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() ?? ''; + try { + // Execute with controlled parallelism + const queue = [...testCases]; + const executing = new Set>(); + + while (queue.length > 0 || executing.size > 0) { + if (cancelledRef.current) break; + + while (queue.length > 0 && executing.size < parallelLimit) { + const tc = queue.shift()!; + const p = executeSingle(tc) + .catch((err) => { + console.error(`Test execution error for ${tc.id}:`, err); + }) + .finally(() => { executing.delete(p); }); + executing.add(p); + } - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - const event: TestEvent = JSON.parse(line.slice(6)); - handleTestEvent(event); - } catch (e) { - console.error('Failed to parse SSE event:', e); - } - } + if (executing.size > 0) { + await Promise.race(executing); } } - // Pass the final results to onComplete - onComplete?.(resultsRef.current); - } catch (err) { - if (err instanceof Error && err.name === 'AbortError') { - console.log('Test execution cancelled'); - } else { - const message = err instanceof Error ? err.message : 'Unknown error'; - setError(message); + if (!cancelledRef.current) { + onComplete?.(resultsRef.current); } + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown error'; + setError(message); } finally { setIsExecuting(false); - abortControllerRef.current = null; + abortControllersRef.current = []; + eventSourcesRef.current = []; } - }, [isExecuting, onComplete, handleTestEvent]); + }, [isExecuting, onComplete]); const cancelExecution = useCallback(() => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); + cancelledRef.current = true; + + // Abort all fetch requests + for (const controller of abortControllersRef.current) { + controller.abort(); + } + + // Close any event sources + for (const eventSource of eventSourcesRef.current) { + eventSource.close(); } }, []);