Skip to content

Commit

Permalink
Feature: AI Executes Callback functions (#88)
Browse files Browse the repository at this point in the history
  • Loading branch information
slavingia authored Nov 26, 2024
2 parents 6d9dd29 + 31d6556 commit 245d04a
Show file tree
Hide file tree
Showing 14 changed files with 236 additions and 112 deletions.
32 changes: 29 additions & 3 deletions app/__tests__/login.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { define, UITestBuilder, expect } from 'shortest';
import { db } from "@/lib/db/drizzle";
import { sql } from "drizzle-orm";
import dotenv from "dotenv";

dotenv.config();

interface LoginState {
username: string;
Expand All @@ -11,7 +16,28 @@ define('Validate login feature implemented with Clerk', async () => {
.given('Github username and password', {
username: process.env.GITHUB_USERNAME || '',
password: process.env.GITHUB_PASSWORD || ''
}, async () => {
expect(process.env.GITHUB_USERNAME).toBeDefined();
})
.when('user is logged in', async () => {
try {
console.log('Starting DB validation...');
const [customer] = await db.execute<{ id: string, name: string, email: string }>(sql`
SELECT * FROM customers WHERE email = 'delba@oliveira.com'
`);

if (!customer) {
throw new Error('Customer delba@oliveira.com not found in database');
}

console.log('Found customer in DB:', customer);
expect(customer.email).toBe('delba@oliveira.com');
expect(customer.name).toBe('Delba de Oliveira');

} catch (error) {
console.error('DB Validation Error:', error);
throw error; // Re-throw to fail the test
}
})
.when('Logged in')
.expect('should redirect to /dashboard');
});
.expect('user should be redirected to /dashboard');
});
23 changes: 23 additions & 0 deletions lib/db/cleanup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { db, client } from "@/lib/db/drizzle";
import { sql } from "drizzle-orm";
import dotenv from "dotenv";

dotenv.config();

async function cleanup() {
try {
console.log('Dropping all tables...');
await db.execute(sql`
DROP TABLE IF EXISTS "pull_requests" CASCADE;
DROP TABLE IF EXISTS "repos" CASCADE;
DROP TABLE IF EXISTS "users" CASCADE;
`);
console.log('All tables dropped successfully');
} catch (error) {
console.error('Error dropping tables:', error);
} finally {
await client.end();
}
}

cleanup();
6 changes: 4 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"db:generate": "drizzle-kit generate",
"db:migrate": "npx tsx lib/db/migrate.ts",
"db:studio": "drizzle-kit studio",
"db:cleanup": "tsx lib/db/cleanup.ts",
"stripe:webhooks": "stripe listen --forward-to http://localhost:3000/api/stripe/webhook",
"test": "shortest"
},
Expand Down Expand Up @@ -78,8 +79,9 @@
"happy-dom": "^15.7.4",
"js-yaml": "^4.1.0",
"jsdom": "^25.0.1",
"shortest": "workspace:*",
"smee-client": "^2.0.3",
"vitest": "^2.1.1",
"shortest": "workspace:*"
"tsx": "^4.7.1",
"vitest": "^2.1.1"
}
}
119 changes: 64 additions & 55 deletions packages/shortest/scripts/test-ai.ts
Original file line number Diff line number Diff line change
@@ -1,78 +1,87 @@
import { BrowserManager } from '../src/core/browser-manager';
import { AIClient } from '../src/ai/ai';
import { BrowserTool } from '../src/browser-use/browser';
import { BrowserManager } from '../src/core/browser-manager';
import { defaultConfig, initialize } from '../src/index';
import { AIClient } from '../src/ai/ai';
import Anthropic from '@anthropic-ai/sdk';
import { ParsedTest } from '../src/types';
import pc from 'picocolors';

async function testBrowser() {
async function testAI() {
console.log(pc.cyan('\n🧪 Testing AI Integration'));
console.log(pc.cyan('======================='));

const browserManager = new BrowserManager();

const apiKey = defaultConfig.ai?.apiKey || process.env.ANTHROPIC_API_KEY;
if (!apiKey) {
console.error(pc.red('Error: Anthropic API key not found in config or environment'));
process.exit(1);
}

try {
await initialize();
console.log(pc.cyan('🚀 Launching browser...'));
console.log('🚀 Launching browser...');
const context = await browserManager.launch();
const page = context.pages()[0];

const browserTool = new BrowserTool(
page,
browserManager,
{
width: 1920,
height: 940
}
);

// Initialize AI client
const aiClient = new AIClient({
apiKey,
model: 'claude-3-5-sonnet-20241022',
maxMessages: 10
});

// Define callbacks with metadata logging
const outputCallback = (content: Anthropic.Beta.Messages.BetaContentBlockParam) => {
if (content.type === 'text') {
console.log(pc.yellow('🤖 Assistant:'), content.text);
}
// Mock test data with callback
const mockTest: ParsedTest = {
suiteName: 'Test Suite',
path: '/',
fullPath: 'http://localhost:3000',
testName: 'Test with callback',
steps: [
{
type: 'GIVEN',
description: 'test setup',
hasCallback: true,
assert: async () => {
console.log('Callback executed: GIVEN step');
}
},
{
type: 'WHEN',
description: 'action performed',
hasCallback: true,
assert: async () => {
console.log('Callback executed: WHEN step');
}
}
]
};

const toolOutputCallback = (name: string, input: any) => {
console.log(pc.yellow('🔧 Tool Use:'), name, input);
if (input.metadata) {
console.log(pc.yellow('Tool Metadata:'), input.metadata);
const browserTool = new BrowserTool(page, browserManager, {
width: 1920,
height: 1080,
testContext: {
currentTest: mockTest,
currentStepIndex: 0,
testName: mockTest.testName
}
};
});

// Test first callback
console.log('\n🔍 Testing first callback:');
const result = await browserTool.execute({
action: 'run_callback'
});
console.log('Result:', result);

// Update test context for second callback
browserTool.updateTestContext({
currentTest: mockTest,
currentStepIndex: 1,
testName: mockTest.testName
});

// Run test
const testPrompt = `Validate the login functionality of the website you see
using github login button
for username argo.mohrad@gmail.com and password: M2@rad99308475
`;

const result = await aiClient.processAction(
testPrompt,
browserTool,
outputCallback,
toolOutputCallback
);
// Test second callback
console.log('\n🔍 Testing second callback:');
const result2 = await browserTool.execute({
action: 'run_callback'
});
console.log('Result:', result2);

console.log(pc.green('✅ Test complete'));

} catch (error) {
console.error(pc.red('❌ Test failed:'), error);
} finally {
console.log(pc.cyan('\n🧹 Cleaning up...'));
console.log('\n🧹 Cleaning up...');
await browserManager.close();
}
}

console.log(pc.cyan('🧪 Browser Integration Test'));
console.log(pc.cyan('==========================='));
testBrowser().catch(console.error);
console.log('🤖 AI Integration Test');
console.log('=====================');
testAI().catch(console.error);
15 changes: 15 additions & 0 deletions packages/shortest/src/ai/ai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,21 @@ export class AIClient {
},
required: ["action"]
}
},
{
name: "run_callback",
description: "Run callback function for current test step",
input_schema: {
type: "object",
properties: {
action: {
type: "string",
enum: ["run_callback"],
description: "Execute callback for current step"
}
},
required: ["action"]
}
}
],
betas: ["computer-use-2024-10-22"]
Expand Down
4 changes: 4 additions & 0 deletions packages/shortest/src/ai/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ Then you can ask for a screenshot to determine for your next action if anything
3. If you need to test a scenario that requires you to test the login flow,
you will need to clear the browser data. For that you can use the "logout" tool that is provided to you via the tools api.
4.IMPORTANT! There is a feature provided to you by tools api called "run_callback" that allows you to run callback functions for a given step.
Whenever you see [HAS_CALLBACK] after the step description, you must call run_callback tool with
the payload provided after you have completed the browser actions for that step.
Your task is to:
1. Execute browser actions to validate test cases
2. Use provided browser tools to interact with the page
Expand Down
25 changes: 23 additions & 2 deletions packages/shortest/src/browser-use/browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { rm } from 'fs/promises';
import { join } from 'path';
import { GitHubTool } from '../tools/github';
import { BrowserManager } from '../core/browser-manager';
import { TestContext, BrowserToolConfig } from '../types/index';

export class BrowserTool extends BaseBrowserTool {
private page: Page;
Expand All @@ -26,6 +27,8 @@ export class BrowserTool extends BaseBrowserTool {
private cursorVisible: boolean = true;
private lastMousePosition: [number, number] = [0, 0];
private githubTool: GitHubTool;
private viewport: { width: number; height: number };
private testContext?: TestContext;

private readonly keyboardShortcuts: Record<string, string | string[]> = {
'ctrl+l': ['Control', 'l'],
Expand Down Expand Up @@ -55,14 +58,16 @@ export class BrowserTool extends BaseBrowserTool {
constructor(
page: Page,
browserManager: BrowserManager,
options: { width: number; height: number; displayNum?: number }
config: BrowserToolConfig
) {
super(options);
super(config);
this.page = page;
this.browserManager = browserManager;
this.screenshotDir = join(process.cwd(), 'screenshots');
mkdirSync(this.screenshotDir, { recursive: true });
this.githubTool = new GitHubTool();
this.viewport = { width: config.width, height: config.height };
this.testContext = config.testContext;

this.initialize();
}
Expand Down Expand Up @@ -237,6 +242,18 @@ export class BrowserTool extends BaseBrowserTool {
metadata: {}
};

case 'run_callback':
if (!this.testContext) {
throw new Error('No test context available for callback execution');
}

const currentStep = this.testContext.currentTest.steps[this.testContext.currentStepIndex];
if (currentStep?.assert) {
await currentStep.assert();
this.testContext.currentStepIndex++;
}
return { output: 'Callback executed successfully' };

default:
throw new ToolError(`Unknown action: ${input.action}`);
}
Expand Down Expand Up @@ -396,4 +413,8 @@ export class BrowserTool extends BaseBrowserTool {
public async waitForNavigation(options?: { timeout: number }): Promise<void> {
await this.page.waitForLoadState('networkidle', { timeout: options?.timeout });
}

updateTestContext(newContext: TestContext) {
this.testContext = newContext;
}
}
2 changes: 1 addition & 1 deletion packages/shortest/src/browser-use/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export interface BrowserToolOptions {
export type ActionInput = {
action: 'mouse_move' | 'left_click' | 'right_click' | 'middle_click' |
'double_click' | 'left_click_drag' | 'cursor_position' |
'screenshot' | 'type' | 'key' | 'github_login' | 'clear_session';
'screenshot' | 'type' | 'key' | 'github_login' | 'clear_session' | 'run_callback';
coordinates?: number[];
text?: string;
username?: string;
Expand Down
11 changes: 10 additions & 1 deletion packages/shortest/src/core/executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { BrowserManager } from './browser-manager';
import { BrowserTool } from '../browser-use/browser';
import { AIClient } from '../ai/ai';
import { initialize, getConfig } from '../index';
import { ParsedTest, TestContext } from '../types';
import Anthropic from '@anthropic-ai/sdk';

interface TestResult {
Expand Down Expand Up @@ -44,6 +45,13 @@ export class TestExecutor {
this.reporter.startSuite(suite.name);

for (const test of suite.tests) {
// Create test context for each test
const testContext: TestContext = {
currentTest: test,
currentStepIndex: 0,
testName: test.testName
};

// Launch new browser for each test
this.reporter.reportStatus('🚀 Launching browser...');
const context = await this.browserManager.launch();
Expand All @@ -54,7 +62,8 @@ export class TestExecutor {
this.browserManager,
{
width: 1920,
height: 1080
height: 1080,
testContext
}
);

Expand Down
Loading

0 comments on commit 245d04a

Please sign in to comment.