fix eval in ci (#90)

* add bananalyzer init to git action workflow

* use headless on ci

* make all remaining evals headless

* add sshx for debugging the eval

* move envs to container level

* debugging evals ci

* only keep last twenty github commits eval for debugging

* update playground

* add more evals back in

* increase ci container size

* revert back to small container

* remove sshx

* add bananalyzer evals back in

* debugging twitter_signup

* testing bigger docker container in ci

* revert back to ubuntu-latest

* >= 20

* remove timeout from git evals

* cleanup

* remove costar + more reliable output checking

* back to ===

* increase trial count to 3

* cleanup playground

* remove trial count 3 until we switch to browserbase (unstable rn)

Author: navidkpr

.github/workflows/ci.yml

@@ -9,6 +9,11 @@ jobs:
   run-evals:
     runs-on: ubuntu-latest
     timeout-minutes: 25
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      HEADLESS: true
 
     steps:
       - name: Check out repository code
@@ -17,7 +22,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: "20"
 
       - name: Install pnpm
         run: npm install -g pnpm
@@ -28,11 +33,13 @@ jobs:
       - name: Install Playwright browsers
         run: pnpm exec playwright install --with-deps
 
+      - name: Initialize Banana Analyzer Evals
+        run: ./evals/bananalyzer-ts/init.sh
+
       - name: Run Evals
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
-          HEADLESS: true
         run: pnpm evals
         timeout-minutes: 20
+
+      # Uncomment the lines below to ssh into the container for debugging
+      # - name: Install sshx
+      #   run: curl -sSf https://sshx.io/get | sh -s run

evals/bananalyzer-ts/index.ts

@@ -59,7 +59,11 @@ export async function evaluateExample(
     return false;
   }
 
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    verbose: 1,
+    headless: process.env.HEADLESS !== "false",
+  });
   await stagehand.init();
 
   let server: Server | null = null;

evals/index.eval.ts

@@ -123,120 +123,76 @@ const peeler_complex = async () => {
 
   return price !== null;
 };
+
 const extract_collaborators_from_github_repository = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    verbose: 1,
+    headless: process.env.HEADLESS !== "false",
+  });
   await stagehand.init();
 
-  const timeoutPromise = new Promise((_, reject) =>
-    setTimeout(() => reject(new Error("Operation timed out")), 60000),
-  );
-
   try {
-    const extractionPromise = (async () => {
-      await stagehand.page.goto("https://github.com/facebook/react");
-      await stagehand.act({
-        action: "find the contributors section",
-      });
-
-      await stagehand.waitForSettledDom();
-
-      const { contributors } = await stagehand.extract({
-        instruction: "Extract top 20 contributors of this repository",
-        schema: z.object({
-          contributors: z.array(
-            z.object({
-              github_username: z.string(),
-              information: z.string(),
-            }),
-          ),
-        }),
-        modelName: "gpt-4o-2024-08-06",
-      });
-
-      console.log("Extracted collaborators:", contributors);
-      return contributors.length === 20;
-    })();
-
-    const result = await Promise.race([extractionPromise, timeoutPromise]);
-    await stagehand.context.close();
-    return result;
-  } catch (error) {
-    console.error("Error or timeout occurred:", error);
-    await stagehand.context.close();
-    return false;
-  }
-};
+    await stagehand.page.goto("https://github.com/facebook/react");
+    await stagehand.act({
+      action: "find the contributors section",
+    });
 
-const extract_last_twenty_github_commits = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
-  await stagehand.init();
+    await stagehand.waitForSettledDom();
 
-  const timeoutPromise = new Promise((_, reject) =>
-    setTimeout(() => reject(new Error("Operation timed out")), 60000),
-  );
+    const { contributors } = await stagehand.extract({
+      instruction: "Extract top 20 contributors of this repository",
+      schema: z.object({
+        contributors: z.array(
+          z.object({
+            github_username: z.string(),
+            information: z.string(),
+          }),
+        ),
+      }),
+      modelName: "gpt-4o-2024-08-06",
+    });
 
-  try {
-    const extractionPromise = (async () => {
-      await stagehand.page.goto("https://github.com/facebook/react");
-
-      await stagehand.waitForSettledDom();
-
-      const { commits } = await stagehand.extract({
-        instruction: "Extract last 20 commits",
-        schema: z.object({
-          commits: z.array(
-            z.object({
-              commit_message: z.string(),
-              commit_url: z.string(),
-              commit_hash: z.string(),
-            }),
-          ),
-        }),
-        modelName: "gpt-4o-2024-08-06",
-      });
-
-      console.log("Extracted commits:", commits);
-      return commits.length === 20;
-    })();
-
-    const result = await Promise.race([extractionPromise, timeoutPromise]);
+    console.log("Extracted collaborators:", contributors);
     await stagehand.context.close();
-    return result;
+    return contributors.length === 20;
   } catch (error) {
     console.error("Error or timeout occurred:", error);
     await stagehand.context.close();
     return false;
   }
 };
 
-const twitter_signup = async () => {
-  const stagehand = new Stagehand({ env: "LOCAL", verbose: 1 });
+const extract_last_twenty_github_commits = async () => {
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    verbose: 1,
+    headless: process.env.HEADLESS !== "false",
+  });
   await stagehand.init();
 
-  const timeoutPromise = new Promise((_, reject) =>
-    setTimeout(() => reject(new Error("Operation timed out")), 120000),
-  );
-
   try {
-    const signupPromise = (async () => {
-      await stagehand.page.goto("https://twitter.com");
-
-      await stagehand.act({
-        action:
-          'sign up with email "{random 12 digit number}@gmail.com", password "TEstTEst.1234". Use whatever else you want for all other fields. You can only stop if you have reached the verification stage.',
-      });
-
-      await stagehand.waitForSettledDom();
+    await stagehand.page.goto("https://github.com/facebook/react");
 
-      // Add a check here to verify if signup was successful
-      // For example, check if a certain element is visible after signup
+    await stagehand.waitForSettledDom();
 
-      return true; // Return true if signup was successful
-    })();
+    const { commits } = await stagehand.extract({
+      instruction: "Extract last 20 commits",
+      schema: z.object({
+        commits: z.array(
+          z.object({
+            commit_message: z.string(),
+            commit_url: z.string(),
+            commit_hash: z.string(),
+          }),
+        ),
+      }),
+      modelName: "gpt-4o-2024-08-06",
+    });
 
-    const result = await Promise.race([signupPromise, timeoutPromise]);
+    console.log("Extracted commits:", commits);
     await stagehand.context.close();
-    return result;
+    return commits.length === 20;
   } catch (error) {
     console.error("Error or timeout occurred:", error);
     await stagehand.context.close();
@@ -396,7 +352,6 @@ const tasks = {
   simple_google_search,
   extract_collaborators_from_github_repository,
   extract_last_twenty_github_commits,
-  twitter_signup,
   costar,
   google_jobs,
 };
@@ -406,7 +361,7 @@ const exactMatch = (args: { input; output; expected? }) => {
 
   return {
     name: "Exact match",
-    score: Boolean(args.output) ? 1 : 0,
+    score: args.output === true || args.output?.success == true,
   };
 };
 
@@ -433,7 +388,6 @@ const testcases = [
   { input: { name: "simple_google_search" } },
   { input: { name: "extract_collaborators_from_github_repository" } },
   { input: { name: "extract_last_twenty_github_commits" } },
-  { input: { name: "twitter_signup" } },
   // { input: { name: "costar" } },
   { input: { name: "google_jobs" } },
   ...chosenBananalyzerEvals.map((evalItem: any) => ({
@@ -496,4 +450,5 @@ Eval("stagehand", {
     }
   },
   scores: [exactMatch],
+  // trialCount: 3,
 });

evals/playground.ts

@@ -1,4 +1,3 @@
-import { Eval } from "braintrust";
 import { Stagehand } from "../lib";
 import { z } from "zod";
 
@@ -116,7 +115,7 @@ async function main() {
   // const [costarResult] = await Promise.all([costar()]);
   const [homedepotResult] = await Promise.all([homedepot()]);
 
-  console.log("Homedepot result:", homedepotResult);
+  console.log("Result:", homedepotResult);
 }
 
 main().catch(console.error);