Use browserbase in github action eval (#84)

* add simple google search eval

* add 2 more evals

* make sure extract continues to use the same model on repeated call

* add twitter sign up eval case

* update eval

* add basic banalayzer eval system

* add server

* update package jsons

* clean up the files

* clean up

cleanup

add gitignore

cleanup

* fix the bananalyzer eval system + add it to the main eval script

* remove all public files on server exit

* fix the package.json playwright issue

* clean up logs

* remove .vscode

* cleanup

* move the test evals to the playground script

* cleanup

* cleanup

* add server/public to gitignore

* test -> playround (much better name)

* fix the resource deletion issue

* update readme + cleanup

* cleanup of readme

* remove the changes in teh lib folder

* cleanup readme

* cleanup

* cleanup

* update readme

* use browserbase browser in github action eval

* force eval env to browserbase for github action

* Set peeler env to local

Author: navidkpr

.github/workflows/ci.yml

@@ -13,7 +13,9 @@ jobs:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
       HEADLESS: true
+      EVAL_ENV: browserbase
 
     steps:
       - name: Check out repository code

evals/index.eval.ts

@@ -5,9 +5,14 @@ import { evaluateExample, chosenBananalyzerEvals } from "./bananalyzer-ts";
 import { createExpressServer } from "./bananalyzer-ts/server/expressServer";
 import process from "process";
 
+const env =
+  process.env.EVAL_ENV?.toLowerCase() === "browserbase"
+    ? "BROWSERBASE"
+    : "LOCAL";
+
 const vanta = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -17,7 +22,10 @@ const vanta = async () => {
 
   const observation = await stagehand.observe("find the request demo button");
 
-  if (!observation) return false;
+  if (!observation) {
+    await stagehand.context.close();
+    return false;
+  }
 
   const observationResult = await stagehand.page
     .locator(stagehand.observations[observation].result)
@@ -38,7 +46,7 @@ const vanta = async () => {
 
 const vanta_h = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -56,7 +64,7 @@ const vanta_h = async () => {
 
 const simple_google_search = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     headless: process.env.HEADLESS !== "false",
   });
   await stagehand.init();
@@ -69,6 +77,7 @@ const simple_google_search = async () => {
 
   const expectedUrl = "https://www.google.com/search?q=OpenAI";
   const currentUrl = await stagehand.page.url();
+
   await stagehand.context.close();
 
   return currentUrl.startsWith(expectedUrl);
@@ -97,7 +106,7 @@ const peeler_simple = async () => {
 
 const peeler_complex = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 1,
     headless: process.env.HEADLESS !== "false",
   });
@@ -202,7 +211,7 @@ const extract_last_twenty_github_commits = async () => {
 
 const wikipedia = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     headless: process.env.HEADLESS !== "false",
   });
@@ -222,7 +231,7 @@ const wikipedia = async () => {
 
 const costar = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     debugDom: true,
     headless: process.env.HEADLESS !== "false",
@@ -270,7 +279,7 @@ const costar = async () => {
 
 const google_jobs = async () => {
   const stagehand = new Stagehand({
-    env: "LOCAL",
+    env,
     verbose: 2,
     debugDom: true,
     headless: process.env.HEADLESS !== "false",