f

anti-work · Jan 16, 2025 · b93e989 · b93e989
1 parent c684eb9
commit b93e989
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 21 deletions.
diff --git a/.github/workflows/shortest.yml b/.github/workflows/shortest.yml
@@ -36,7 +36,7 @@ jobs:
           cp .vercel/.env.preview.local .env.local
           test -f .env.local || (echo ".env.local not created" && exit 1)
 
-      - name: Setup database
+      - name: Set up database
         run: |
           pnpm drizzle-kit generate
           pnpm db:migrate

diff --git a/packages/shortest/src/core/runner/index.ts b/packages/shortest/src/core/runner/index.ts
@@ -24,6 +24,7 @@ import { TestCompiler } from "../compiler";
 interface TestResult {
   result: "pass" | "fail";
   reason: string;
+  tokenUsage?: { input: number; output: number };
 }
 
 export class TestRunner {
@@ -180,7 +181,11 @@ export class TestRunner {
 
     // this may never happen as the config is initialized before this code is executed
     if (!this.config.anthropicKey) {
-      throw new Error("ANTHROPIC_KEY is not set");
+      return {
+        result: "fail" as const,
+        reason: "ANTHROPIC_KEY is not set",
+        tokenUsage: { input: 0, output: 0 },
+      };
     }
 
     const aiClient = new AIClient(
@@ -280,11 +285,7 @@ export class TestRunner {
     const result = await aiClient.processAction(prompt, browserTool);
 
     if (!result) {
-      return {
-        result: "fail" as const,
-        reason: "AI processing failed: no result returned",
-        tokenUsage: { input: 0, output: 0 },
-      };
+      throw new Error("AI processing failed: no result returned");
     }
 
     // Parse AI result first
@@ -297,22 +298,14 @@ export class TestRunner {
     );
 
     if (!finalMessage || finalMessage.type !== "text") {
-      return {
-        result: "fail" as const,
-        reason: "No test result found in AI response",
-        tokenUsage: result.tokenUsage,
-      };
+      throw new Error("No test result found in AI response");
     }
 
     const jsonMatch = (
       finalMessage as Anthropic.Beta.Messages.BetaTextBlock
     ).text.match(/{[\s\S]*}/);
     if (!jsonMatch) {
-      return {
-        result: "fail" as const,
-        reason: "Invalid test result format",
-        tokenUsage: result.tokenUsage,
-      };
+      throw new Error("Invalid test result format");
     }
 
     const aiResult = JSON.parse(jsonMatch[0]) as TestResult;
@@ -449,7 +442,11 @@ export class TestRunner {
       );
 
     if (!steps) {
-      throw new Error("No steps to execute, running test in normal mode");
+      return {
+        result: "fail" as const,
+        reason: "No steps to execute, running test in normal mode",
+        tokenUsage: { input: 0, output: 0 },
+      };
     }
     for (const step of steps) {
       await new Promise((resolve) => setTimeout(resolve, 1000));
@@ -465,9 +462,12 @@ export class TestRunner {
           await browserTool.getNormalizedComponentStringByCoords(x, y);
 
         if (componentStr !== step.extras.componentStr) {
-          throw new Error(
-            "Component UI elements are different, running test in normal mode",
-          );
+          return {
+            result: "fail" as const,
+            reason:
+              "Component UI elements are different, running test in normal mode",
+            tokenUsage: { input: 0, output: 0 },
+          };
         }
       }
       if (step.action?.input) {