Feature: Implement Assertion (#85)

Add assertion support and DB validation to test framework

Changes:
- Added Jest expect integration for assertions
- Added support for async DB validations in test steps
- Improved error handling and reporting for assertions
- Added test script to verify assertion implementation
- Updated test builder to handle both sync/async test execution

Example usage:

```typescript
.when('Logged in', async () => {
const user = await db.query.users.findFirst();
expect(user).toBeDefined();
})
```

Demo: 

![image](https://github.com/user-attachments/assets/7bcd4e13-8d29-476b-b8bb-2c03d9431681)

README.md

@@ -161,20 +161,19 @@ or use the `pnpm` command:
 pnpm test
 ```
 
-## Test Github MFA Login Flow with Shortest
-In order to test Github MFA login in browser you need to add register shortest as OTP provider and add the OTP secret to your `.env.local` file:
-
-1. Go to your repo settings
-2. Navigate to `Password and Authentication`
-3. Click on `Authenticator App`
-4. Choose `Use your authenticator app`
-5. Click on `Setup key` to grab the OTP secret
-6. Add the OTP secret to your `.env.local` file or use the `shortest` cli to add it:
-
+## Testing Github MFA Login Flow with Shortest
+To test Github MFA login in a browser, you need to register Shortest as an OTP provider and add the OTP secret to your `.env.local` file:
+
+1. Go to your repository settings
+2. Navigate to "Password and Authentication"
+3. Click on "Authenticator App"
+4. Select "Use your authenticator app"
+5. Click "Setup key" to obtain the OTP secret
+6. Add the OTP secret to your `.env.local` file or use the Shortest CLI to add it
+7. Enter the 2FA code displayed in your terminal into Github's Authenticator setup page to complete the process
 ```bash
 shortest --github-code --secret=<OTP_SECRET>
 ```
-7. Laslty enter the 2FA code generated in terminal in your Github Authenticator editor
 
 
 ## To run tests in Github workflows

app/__tests__/assert.test.ts

@@ -0,0 +1,50 @@
+import { define, UITestBuilder, expect } from 'shortest';
+import { db } from "@/lib/db/drizzle";
+
+interface User {
+  email: string;
+  name: string;
+}
+
+define('Test Assertions', async () => {
+  // Test 1: Basic Assertions (Will Pass)
+   new UITestBuilder<User>('/')
+    .test('Basic assertions that pass')
+    .given('a test user', { email: 'test@test.com', name: 'Test User' }, async () => {
+      expect(true).toBe(true);
+      expect({ foo: 'bar' }).toEqual({ foo: 'bar' });
+      expect([1, 2, 3]).toContain(2);
+    })
+    .when('checking database', async () => {
+      const mockUser = { id: 1, email: 'test@test.com' };
+      expect(mockUser).toHaveProperty('email');
+      expect(mockUser.email).toMatch(/test@test.com/);
+    })
+    .expect('all assertions to pass');
+
+  // Test 2: Failing Assertions (Will Fail)
+new UITestBuilder<User>('/')
+    .test('Assertions that should fail')
+    .given('some data', { email: 'fail@test.com', name: 'Fail Test' }, async () => {
+      expect(true).toBe(false);
+      expect({ foo: 'bar' }).toEqual({ foo: 'baz' });
+    })
+    .when('checking values', async () => {
+      expect(null).toBeDefined();
+      expect(undefined).toBeTruthy();
+    })
+    .expect('to see failure reports');
+
+  // Test 3: Async Assertions (Mix of Pass/Fail)
+new UITestBuilder<User>('/')
+    .test('Async assertions')
+    .given('database connection', async () => {
+      const user = await db.query.users.findFirst();
+      expect(user).toBeDefined();
+    })
+    .when('querying data', async () => {
+      const result = await Promise.resolve({ status: 'error' });
+      expect(result.status).toBe('success');
+    })
+    .expect('to see mix of pass/fail results');
+});

app/__tests__/home.test.ts

@@ -4,12 +4,10 @@ interface loginButton {
   url: string;
 }
 
-define('Validate Dasboard is accessible by users', () => {
-
+define('Validate Dasboard is accessible by users', async () => {
   new UITestBuilder<loginButton>('/')
     .test('Validate that users can access the dashboard')
     .given('baseUrl', { url: 'http://localhost:3000' })
     .when('Clicking on view dashboard button')
-    .expect('Should be able redirect to /dashboard and see the dashboard')
-
+    .expect('Should be able redirect to /dashboard and see the dashboard');
 });

app/__tests__/login.test.ts

@@ -1,21 +1,17 @@
-import { afterAll, beforeAll, define, UITestBuilder } from 'shortest';
+import { define, UITestBuilder, expect } from 'shortest';
 
 interface LoginState {
   username: string;
   password: string;
 }
 
-define('Validate login feature implemented with Clerk', () => {
-
-  // new UITestBuilder<LoginState>('/')
-    // .test('Login to the app using Email and Password')
-    // .given('username and password', { username: 'argo.mohrad@gmail.com', password: 'Shortest1234' })
-    // .when('Logged in, click on view dashboard button')
-    // .expect('should successfully redirect to /')
-
-    new UITestBuilder<LoginState>('/')
+define('Validate login feature implemented with Clerk', async () => {
+  new UITestBuilder<LoginState>('/')
     .test('Login to the app using Github login')
-    .given('Github username and password', { username: `${process.env.GITHUB_USERNAME}`, password: `${process.env.GITHUB_PASSWORD}` })
-    .expect('should successfully redirect to /dashboard')
-
+    .given('Github username and password', { 
+      username: process.env.GITHUB_USERNAME || '',
+      password: process.env.GITHUB_PASSWORD || ''
+    })
+    .when('Logged in')
+    .expect('should redirect to /dashboard');
 });

packages/shortest/package.json

@@ -31,7 +31,8 @@
     "test:ai": "tsx scripts/test-ai.ts",
     "test:browser": "tsx scripts/test-browser.ts",
     "test:coordinates": "tsx scripts/test-coordinates.ts",
-    "test:github": "tsx scripts/test-github.ts"
+    "test:github": "tsx scripts/test-github.ts",
+    "test:assertion": "tsx scripts/test-assertion.ts"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "0.32.0",
@@ -43,11 +44,13 @@
     "dotenv": "^16.4.5",
     "otplib": "^12.0.1",
     "picocolors": "^1.0.0",
-    "playwright": "^1.42.1"
+    "playwright": "^1.42.1",
+    "expect": "^29.7.0"
   },
   "devDependencies": {
     "@types/node": "^20.11.24",
     "tsx": "^4.7.1",
-    "typescript": "~5.6.2"
+    "typescript": "~5.6.2",
+    "@types/jest": "^29.5.12"
   }
 }

packages/shortest/scripts/test-assertion.ts

@@ -0,0 +1,72 @@
+import { UITestBuilder, expect } from '../src/index';
+import pc from 'picocolors';
+
+async function testAssertions() {
+  console.log(pc.cyan('\n🧪 Testing Assertion Implementation'));
+  console.log(pc.cyan('================================'));
+
+  let failedTests = 0;
+  let passedTests = 0;
+
+  try {
+    // Test 1: Verify failing assertions are caught
+    console.log(pc.cyan('\nTest 1: Verify failing assertions'));
+    try {
+      const builder = new UITestBuilder('/')
+        .test('Test failing assertion')
+        .given('test data', undefined, async () => {
+          expect(true).toBe(false);
+        });
+
+      console.log(pc.red('❌ Failed: Assertion should have thrown error'));
+      failedTests++;
+    } catch (error) {
+      console.log(pc.green('✅ Passed: Caught failing assertion'));
+      passedTests++;
+    }
+
+    // Test 2: Verify async assertions
+    console.log(pc.cyan('\nTest 2: Verify async assertions'));
+    try {
+      const builder = new UITestBuilder('/')
+        .test('Test async assertion')
+        .given('test data', undefined, async () => {
+          const result = await Promise.resolve(false);
+          expect(result).toBe(true);
+        });
+
+      console.log(pc.red('❌ Failed: Async assertion should have thrown'));
+      failedTests++;
+    } catch (error) {
+      console.log(pc.green('✅ Passed: Caught async failing assertion'));
+      passedTests++;
+    }
+
+    // Test 3: Verify assertion steps are recorded
+    console.log(pc.cyan('\nTest 3: Verify assertion recording'));
+    const builder = new UITestBuilder('/')
+      .test('Test step recording')
+      .given('test data', undefined, async () => {
+        expect(true).toBe(true);
+      });
+
+    if (builder.steps.length === 1 && builder.steps[0].assert) {
+      console.log(pc.green('✅ Passed: Assertion step recorded'));
+      passedTests++;
+    } else {
+      console.log(pc.red('❌ Failed: Assertion step not recorded'));
+      failedTests++;
+    }
+
+    // Summary
+    console.log(pc.cyan('\n📊 Test Summary'));
+    console.log(pc.cyan('============='));
+    console.log(pc.green(`Passed: ${passedTests}`));
+    console.log(pc.red(`Failed: ${failedTests}`));
+
+  } catch (error) {
+    console.error(pc.red('\n❌ Test script failed:'), error);
+  }
+}
+
+testAssertions().catch(console.error);

packages/shortest/src/ai/ai.ts

@@ -21,6 +21,28 @@ export class AIClient {
     browserTool: BrowserTool,
     outputCallback?: (content: Anthropic.Beta.Messages.BetaContentBlockParam) => void,
     toolOutputCallback?: (name: string, input: any) => void
+  ) {
+    const maxRetries = 3;
+    let attempts = 0;
+
+    while (attempts < maxRetries) {
+      try {
+        return await this.makeRequest(prompt, browserTool, outputCallback, toolOutputCallback);
+      } catch (error: any) {
+        attempts++;
+        if (attempts === maxRetries) throw error;
+
+        console.log(`Retry attempt ${attempts}/${maxRetries}`);
+        await new Promise(r => setTimeout(r, 5000 * attempts));
+      }
+    }
+  }
+
+  async makeRequest(
+    prompt: string,
+    browserTool: BrowserTool,
+    outputCallback?: (content: Anthropic.Beta.Messages.BetaContentBlockParam) => void,
+    toolOutputCallback?: (name: string, input: any) => void
   ) {
     const messages: Anthropic.Beta.Messages.BetaMessageParam[] = [];
 

packages/shortest/src/core/executor.ts

@@ -84,6 +84,10 @@ export class TestExecutor {
               }
             );
 
+            if (!result) {
+              throw new Error('AI processing failed: no result returned');
+            }
+
             // Parse final response for JSON result
             const finalMessage = result.finalResponse.content.find(block => 
               block.type === 'text' && 

packages/shortest/src/core/parser.ts

@@ -25,7 +25,6 @@ export class TestParser {
         tests: builders.map((builder: UITestBuilderInterface) => this.parseTestBuilder(builder))
       };
 
-      console.log(`Test Suite: ${suiteName}`);
       suite.tests.forEach(test => this.generateTestPrompt(test, suiteName));
 
       this.processedSuites.add(suiteName);

packages/shortest/src/core/reporter.ts

@@ -1,5 +1,5 @@
 import pc from 'picocolors';
-import { ParsedTestSuite } from '../types';
+import { ParsedTestSuite, AssertionError } from '../types';
 
 export type TestStatus = 'pending' | 'running' | 'passed' | 'failed';
 
@@ -103,4 +103,21 @@ export class Reporter {
   reportError(context: string, message: string) {
     console.error(pc.red(`\n${context} Error: ${message}`));
   }
+
+  reportAssertion(
+    step: string, 
+    status: 'passed' | 'failed', 
+    error?: AssertionError
+  ): void {
+    const icon = status === 'passed' ? '✓' : '✗';
+    const color = status === 'passed' ? 'green' : 'red';
+
+    console.log(pc[color](`${icon} ${step}`));
+
+    if (error && status === 'failed') {
+      console.log(pc.red(`  Expected: ${error.matcherResult?.expected}`));
+      console.log(pc.red(`  Received: ${error.matcherResult?.actual}`));
+      console.log(pc.red(`  Message: ${error.message}`));
+    }
+  }
 }

packages/shortest/src/index.ts

@@ -7,6 +7,7 @@ import { UITestBuilderInterface } from './types/builder';
 import { BeforeAllFunction, AfterAllFunction, TestSuite } from './types';
 import dotenv from 'dotenv';
 import { join } from 'path';
+import { expect as jestExpect } from 'expect';
 
 // Define global registry type
 declare global {
@@ -100,10 +101,11 @@ export class TestRegistry {
 }
 
 // Export test functions
-export function define(name: string, fn: () => void): void {
+export function define(name: string, fn: () => void | Promise<void>): void {
   TestRegistry.startSuite(name);
-  fn();
-  TestRegistry.endSuite();
+  Promise.resolve(fn()).then(() => {
+    TestRegistry.endSuite();
+  });
 }
 
 export function beforeAll(fn: BeforeAllFunction): void {
@@ -121,3 +123,4 @@ export { UITestBuilder };
 export type { UITestBuilderInterface };
 export type { ShortestConfig };
 export * from './types';
+export { jestExpect as expect };

packages/shortest/src/types/builder.ts

@@ -7,9 +7,9 @@ export interface UITestBuilderInterface<T = any> {
   setSuiteName(name: string): this;
   getSuiteName(): string;
   test(name: string): this;
-  given(actionOrState: ActionType, payload?: T): this;
-  when(action: ActionType, payload?: T): this;
-  expect(assertion: ActionType, payload?: T): this;
+  given(action: string, payload?: T | (() => Promise<void>) | undefined, assert?: () => Promise<void>): this;
+  when(action: string, payload?: T | (() => Promise<void>) | undefined, assert?: () => Promise<void>): this;
+  expect(assertion: string, payload?: T | (() => Promise<void>) | undefined, assert?: () => Promise<void>): this;
   before(actionOrFn: ActionType | BeforeAllFunction, payload?: T): this;
   after(actionOrFn: ActionType | AfterAllFunction, payload?: T): this;
 } 

packages/shortest/src/types/index.ts

@@ -24,6 +24,7 @@ export interface TestStep {
     type: 'GIVEN' | 'WHEN' | 'EXPECT' | 'BEFORE' | 'AFTER';
     action: string;
     payload?: any;
+    assert?: () => Promise<void>;
 }
 
 // Browser related types
@@ -74,4 +75,13 @@ export interface ParsedTest {
 export interface ParsedTestSuite {
     name: string;
     tests: ParsedTest[];
+}
+
+export interface AssertionError extends Error {
+  matcherResult?: {
+    message: string;
+    pass: boolean;
+    actual: any;
+    expected: any;
+  };
 }