From a6636d75de260342233d79deb6a0b1a551cecd0f Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:05 -0800 Subject: [PATCH 01/12] feat: add assertion testing infrastructure and test cases --- app/__tests__/assert.test.ts | 50 ++++++++++++++ packages/shortest/scripts/test-assertion.ts | 72 +++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 app/__tests__/assert.test.ts create mode 100644 packages/shortest/scripts/test-assertion.ts diff --git a/app/__tests__/assert.test.ts b/app/__tests__/assert.test.ts new file mode 100644 index 00000000..38c29e96 --- /dev/null +++ b/app/__tests__/assert.test.ts @@ -0,0 +1,50 @@ +import { define, UITestBuilder, expect } from 'shortest'; +import { db } from "@/lib/db/drizzle"; + +interface User { + email: string; + name: string; +} + +define('Test Assertions', async () => { + // Test 1: Basic Assertions (Will Pass) + new UITestBuilder('/') + .test('Basic assertions that pass') + .given('a test user', { email: 'test@test.com', name: 'Test User' }, async () => { + expect(true).toBe(true); + expect({ foo: 'bar' }).toEqual({ foo: 'bar' }); + expect([1, 2, 3]).toContain(2); + }) + .when('checking database', async () => { + const mockUser = { id: 1, email: 'test@test.com' }; + expect(mockUser).toHaveProperty('email'); + expect(mockUser.email).toMatch(/test@test.com/); + }) + .expect('all assertions to pass'); + + // Test 2: Failing Assertions (Will Fail) +new UITestBuilder('/') + .test('Assertions that should fail') + .given('some data', { email: 'fail@test.com', name: 'Fail Test' }, async () => { + expect(true).toBe(false); + expect({ foo: 'bar' }).toEqual({ foo: 'baz' }); + }) + .when('checking values', async () => { + expect(null).toBeDefined(); + expect(undefined).toBeTruthy(); + }) + .expect('to see failure reports'); + + // Test 3: Async Assertions (Mix of Pass/Fail) +new UITestBuilder('/') + .test('Async assertions') + .given('database connection', async () => { + const user = await db.query.users.findFirst(); + expect(user).toBeDefined(); + }) + .when('querying data', async () => { + const result = await Promise.resolve({ status: 'error' }); + expect(result.status).toBe('success'); + }) + .expect('to see mix of pass/fail results'); +}); diff --git a/packages/shortest/scripts/test-assertion.ts b/packages/shortest/scripts/test-assertion.ts new file mode 100644 index 00000000..3a44abb7 --- /dev/null +++ b/packages/shortest/scripts/test-assertion.ts @@ -0,0 +1,72 @@ +import { UITestBuilder, expect } from '../src/index'; +import pc from 'picocolors'; + +async function testAssertions() { + console.log(pc.cyan('\n๐Ÿงช Testing Assertion Implementation')); + console.log(pc.cyan('================================')); + + let failedTests = 0; + let passedTests = 0; + + try { + // Test 1: Verify failing assertions are caught + console.log(pc.cyan('\nTest 1: Verify failing assertions')); + try { + const builder = new UITestBuilder('/') + .test('Test failing assertion') + .given('test data', undefined, async () => { + expect(true).toBe(false); + }); + + console.log(pc.red('โŒ Failed: Assertion should have thrown error')); + failedTests++; + } catch (error) { + console.log(pc.green('โœ… Passed: Caught failing assertion')); + passedTests++; + } + + // Test 2: Verify async assertions + console.log(pc.cyan('\nTest 2: Verify async assertions')); + try { + const builder = new UITestBuilder('/') + .test('Test async assertion') + .given('test data', undefined, async () => { + const result = await Promise.resolve(false); + expect(result).toBe(true); + }); + + console.log(pc.red('โŒ Failed: Async assertion should have thrown')); + failedTests++; + } catch (error) { + console.log(pc.green('โœ… Passed: Caught async failing assertion')); + passedTests++; + } + + // Test 3: Verify assertion steps are recorded + console.log(pc.cyan('\nTest 3: Verify assertion recording')); + const builder = new UITestBuilder('/') + .test('Test step recording') + .given('test data', undefined, async () => { + expect(true).toBe(true); + }); + + if (builder.steps.length === 1 && builder.steps[0].assert) { + console.log(pc.green('โœ… Passed: Assertion step recorded')); + passedTests++; + } else { + console.log(pc.red('โŒ Failed: Assertion step not recorded')); + failedTests++; + } + + // Summary + console.log(pc.cyan('\n๐Ÿ“Š Test Summary')); + console.log(pc.cyan('=============')); + console.log(pc.green(`Passed: ${passedTests}`)); + console.log(pc.red(`Failed: ${failedTests}`)); + + } catch (error) { + console.error(pc.red('\nโŒ Test script failed:'), error); + } +} + +testAssertions().catch(console.error); From d61f6e900ab3a1f165694ca3bc603bd272711064 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:09 -0800 Subject: [PATCH 02/12] feat: add support for async assertions in test builder --- packages/shortest/src/types/builder.ts | 6 +-- packages/shortest/src/ui-test-builder.ts | 69 +++++++++++++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/packages/shortest/src/types/builder.ts b/packages/shortest/src/types/builder.ts index 5fbc1ece..b698fb83 100644 --- a/packages/shortest/src/types/builder.ts +++ b/packages/shortest/src/types/builder.ts @@ -7,9 +7,9 @@ export interface UITestBuilderInterface { setSuiteName(name: string): this; getSuiteName(): string; test(name: string): this; - given(actionOrState: ActionType, payload?: T): this; - when(action: ActionType, payload?: T): this; - expect(assertion: ActionType, payload?: T): this; + given(action: string, payload?: T | (() => Promise) | undefined, assert?: () => Promise): this; + when(action: string, payload?: T | (() => Promise) | undefined, assert?: () => Promise): this; + expect(assertion: string, payload?: T | (() => Promise) | undefined, assert?: () => Promise): this; before(actionOrFn: ActionType | BeforeAllFunction, payload?: T): this; after(actionOrFn: ActionType | AfterAllFunction, payload?: T): this; } \ No newline at end of file diff --git a/packages/shortest/src/ui-test-builder.ts b/packages/shortest/src/ui-test-builder.ts index 7721c49a..215e0e1c 100644 --- a/packages/shortest/src/ui-test-builder.ts +++ b/packages/shortest/src/ui-test-builder.ts @@ -11,7 +11,7 @@ export class UITestBuilder implements UITestBuilderInterface { constructor(path: string) { this.path = path; this.testName = ''; - TestRegistry.registerTest(this as UITestBuilderInterface); + TestRegistry.registerTest(this as unknown as UITestBuilderInterface); } test(name: string): this { @@ -28,50 +28,67 @@ export class UITestBuilder implements UITestBuilderInterface { return this.suiteName; } - given(actionOrState: ActionType, payload?: T): this { - if (typeof actionOrState === 'string') { - return this.addStep('GIVEN', actionOrState, payload); + given(action?: string, payload?: T | (() => Promise), assert?: () => Promise): this { + if (action) { + if (payload && typeof payload === 'function') { + const assertFn = payload as () => Promise; + this.addStep('GIVEN', action, undefined, assertFn); + } else { + this.addStep('GIVEN', action, payload as T, assert); + } } - return this.addStep('GIVEN', 'SET_STATE', actionOrState); + return this; } - when(action: ActionType, payload?: T): this { - if (typeof action === 'string') { - return this.addStep('WHEN', action, payload); + when(action?: string, payload?: T | (() => Promise), assert?: () => Promise): this { + if (action) { + if (payload && typeof payload === 'function') { + const assertFn = payload as () => Promise; + this.addStep('WHEN', action, undefined, assertFn); + } else { + this.addStep('WHEN', action, payload as T, assert); + } } - return this.addStep('WHEN', 'SET_STATE', action); + return this; } - expect(assertion: ActionType, payload?: T): this { - if (typeof assertion === 'string') { - return this.addStep('EXPECT', assertion, payload); + expect(assertion?: string, payload?: T | (() => Promise), assert?: () => Promise): this { + if (assertion) { + if (payload && typeof payload === 'function') { + const assertFn = payload as () => Promise; + this.addStep('EXPECT', assertion, undefined, assertFn); + } else { + this.addStep('EXPECT', assertion, payload as T, assert); + } } - return this.addStep('EXPECT', 'ASSERT_STATE', assertion); + return this; } before(actionOrFn: ActionType | BeforeAllFunction, payload?: T): this { if (typeof actionOrFn === 'function') { - return this.addStep('BEFORE', 'EXECUTE_FUNCTION', actionOrFn); - } - if (typeof actionOrFn === 'string') { - return this.addStep('BEFORE', actionOrFn, payload); + this.addStep('BEFORE', 'EXECUTE_FUNCTION', actionOrFn); + } else { + this.addStep('BEFORE', typeof actionOrFn === 'string' ? actionOrFn : 'SET_STATE', payload); } - return this.addStep('BEFORE', 'SET_STATE', actionOrFn); + return this; } after(actionOrFn: ActionType | AfterAllFunction, payload?: T): this { if (typeof actionOrFn === 'function') { - return this.addStep('AFTER', 'EXECUTE_FUNCTION', actionOrFn); - } - if (typeof actionOrFn === 'string') { - return this.addStep('AFTER', actionOrFn, payload); + this.addStep('AFTER', 'EXECUTE_FUNCTION', actionOrFn); + } else { + this.addStep('AFTER', typeof actionOrFn === 'string' ? actionOrFn : 'SET_STATE', payload); } - return this.addStep('AFTER', 'SET_STATE', actionOrFn); + return this; } - private addStep(type: TestStep['type'], action: string, payload?: any): this { - this.steps.push({ type, action, payload }); - return this; + private addStep( + type: TestStep['type'], + action: string, + payload?: any, + assert?: () => Promise + ): void { + this.steps.push({ type, action, payload, assert }); } } From 7ca99c1c73235af502359cc7a10809fa0d1300c0 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:13 -0800 Subject: [PATCH 03/12] feat: integrate jest expect and improve error handling --- packages/shortest/src/core/executor.ts | 4 ++++ packages/shortest/src/index.ts | 9 ++++++--- packages/shortest/src/types/index.ts | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/shortest/src/core/executor.ts b/packages/shortest/src/core/executor.ts index 50f165df..7e5bdbe7 100644 --- a/packages/shortest/src/core/executor.ts +++ b/packages/shortest/src/core/executor.ts @@ -84,6 +84,10 @@ export class TestExecutor { } ); + if (!result) { + throw new Error('AI processing failed: no result returned'); + } + // Parse final response for JSON result const finalMessage = result.finalResponse.content.find(block => block.type === 'text' && diff --git a/packages/shortest/src/index.ts b/packages/shortest/src/index.ts index 5eee6249..0ec47088 100644 --- a/packages/shortest/src/index.ts +++ b/packages/shortest/src/index.ts @@ -7,6 +7,7 @@ import { UITestBuilderInterface } from './types/builder'; import { BeforeAllFunction, AfterAllFunction, TestSuite } from './types'; import dotenv from 'dotenv'; import { join } from 'path'; +import { expect as jestExpect } from 'expect'; // Define global registry type declare global { @@ -100,10 +101,11 @@ export class TestRegistry { } // Export test functions -export function define(name: string, fn: () => void): void { +export function define(name: string, fn: () => void | Promise): void { TestRegistry.startSuite(name); - fn(); - TestRegistry.endSuite(); + Promise.resolve(fn()).then(() => { + TestRegistry.endSuite(); + }); } export function beforeAll(fn: BeforeAllFunction): void { @@ -121,3 +123,4 @@ export { UITestBuilder }; export type { UITestBuilderInterface }; export type { ShortestConfig }; export * from './types'; +export { jestExpect as expect }; diff --git a/packages/shortest/src/types/index.ts b/packages/shortest/src/types/index.ts index 6372c342..db49658a 100644 --- a/packages/shortest/src/types/index.ts +++ b/packages/shortest/src/types/index.ts @@ -24,6 +24,7 @@ export interface TestStep { type: 'GIVEN' | 'WHEN' | 'EXPECT' | 'BEFORE' | 'AFTER'; action: string; payload?: any; + assert?: () => Promise; } // Browser related types From 36c4fd46fdcc0c5fd2872bfc67d036e4d862e007 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:18 -0800 Subject: [PATCH 04/12] refactor: update test files to use new assertion syntax --- app/__tests__/home.test.ts | 6 ++---- app/__tests__/login.test.ts | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/__tests__/home.test.ts b/app/__tests__/home.test.ts index c8fa1b78..40906c92 100644 --- a/app/__tests__/home.test.ts +++ b/app/__tests__/home.test.ts @@ -4,12 +4,10 @@ interface loginButton { url: string; } -define('Validate Dasboard is accessible by users', () => { - +define('Validate Dasboard is accessible by users', async () => { new UITestBuilder('/') .test('Validate that users can access the dashboard') .given('baseUrl', { url: 'http://localhost:3000' }) .when('Clicking on view dashboard button') - .expect('Should be able redirect to /dashboard and see the dashboard') - + .expect('Should be able redirect to /dashboard and see the dashboard'); }); diff --git a/app/__tests__/login.test.ts b/app/__tests__/login.test.ts index 7852499a..92ef82d9 100644 --- a/app/__tests__/login.test.ts +++ b/app/__tests__/login.test.ts @@ -1,21 +1,23 @@ -import { afterAll, beforeAll, define, UITestBuilder } from 'shortest'; +import { afterAll, beforeAll, define, UITestBuilder, expect } from 'shortest'; +import { client, db } from "@/lib/db/drizzle"; interface LoginState { username: string; password: string; } -define('Validate login feature implemented with Clerk', () => { - - // new UITestBuilder('/') - // .test('Login to the app using Email and Password') - // .given('username and password', { username: 'argo.mohrad@gmail.com', password: 'Shortest1234' }) - // .when('Logged in, click on view dashboard button') - // .expect('should successfully redirect to /') - - new UITestBuilder('/') +define('Validate login feature implemented with Clerk', async () => { + new UITestBuilder('/') .test('Login to the app using Github login') - .given('Github username and password', { username: `${process.env.GITHUB_USERNAME}`, password: `${process.env.GITHUB_PASSWORD}` }) - .expect('should successfully redirect to /dashboard') - + .given('Github username and password', { + username: process.env.GITHUB_USERNAME || '', + password: process.env.GITHUB_PASSWORD || '' + }, async () => { + expect(process.env.GITHUB_USERNAME).toBeDefined(); + }) + .when('Logged in', async () => { + const user = await db.query.users.findFirst(); + expect(user).toBeDefined(); + }) + .expect('should redirect to /dashboard'); }); \ No newline at end of file From d83b1892e93d35d5168c425db23daba7a71c30d4 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:22 -0800 Subject: [PATCH 05/12] chore: add jest dependencies and update typescript config --- packages/shortest/package.json | 9 +- packages/shortest/tsconfig.json | 3 +- pnpm-lock.yaml | 200 ++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 4 deletions(-) diff --git a/packages/shortest/package.json b/packages/shortest/package.json index 8d34db1d..e1e7f2b4 100644 --- a/packages/shortest/package.json +++ b/packages/shortest/package.json @@ -31,7 +31,8 @@ "test:ai": "tsx scripts/test-ai.ts", "test:browser": "tsx scripts/test-browser.ts", "test:coordinates": "tsx scripts/test-coordinates.ts", - "test:github": "tsx scripts/test-github.ts" + "test:github": "tsx scripts/test-github.ts", + "test:assertion": "tsx scripts/test-assertion.ts" }, "dependencies": { "@anthropic-ai/sdk": "0.32.0", @@ -43,11 +44,13 @@ "dotenv": "^16.4.5", "otplib": "^12.0.1", "picocolors": "^1.0.0", - "playwright": "^1.42.1" + "playwright": "^1.42.1", + "expect": "^29.7.0" }, "devDependencies": { "@types/node": "^20.11.24", "tsx": "^4.7.1", - "typescript": "~5.6.2" + "typescript": "~5.6.2", + "@types/jest": "^29.5.12" } } \ No newline at end of file diff --git a/packages/shortest/tsconfig.json b/packages/shortest/tsconfig.json index f26c4491..bc6ad809 100644 --- a/packages/shortest/tsconfig.json +++ b/packages/shortest/tsconfig.json @@ -8,7 +8,8 @@ "declaration": true, "esModuleInterop": true, "skipLibCheck": true, - "rootDir": "./src" + "rootDir": "./src", + "types": ["node", "jest"] }, "include": ["src"], "exclude": ["node_modules", "dist"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d111605c..43a5f62a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -222,6 +222,9 @@ importers: esbuild: specifier: ^0.20.1 version: 0.20.2 + expect: + specifier: ^29.7.0 + version: 29.7.0 glob: specifier: ^10.3.10 version: 10.4.5 @@ -235,6 +238,9 @@ importers: specifier: ^1.42.1 version: 1.48.2 devDependencies: + '@types/jest': + specifier: ^29.5.12 + version: 29.5.14 '@types/node': specifier: ^20.11.24 version: 20.17.3 @@ -1378,6 +1384,18 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@jest/expect-utils@29.7.0': + resolution: {integrity: sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/schemas@29.6.3': + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/types@29.6.3': + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jridgewell/gen-mapping@0.3.5': resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} engines: {node: '>=6.0.0'} @@ -2197,6 +2215,9 @@ packages: cpu: [x64] os: [win32] + '@sinclair/typebox@0.27.8': + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + '@swc/core-darwin-arm64@1.7.42': resolution: {integrity: sha512-fWhaCs2+8GDRIcjExVDEIfbptVrxDqG8oHkESnXgymmvqTWzWei5SOnPNMS8Q+MYsn/b++Y2bDxkcwmq35Bvxg==} engines: {node: '>=10'} @@ -2343,6 +2364,18 @@ packages: '@types/hoist-non-react-statics@3.3.5': resolution: {integrity: sha512-SbcrWzkKBw2cdwRTwQAswfpB9g9LJWfjtUeW/jvNwbhC8cpmmNYVePa+ncbUe0rGTQ7G3Ff6mYUN2VMfLVr+Sg==} + '@types/istanbul-lib-coverage@2.0.6': + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + + '@types/istanbul-lib-report@3.0.3': + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + + '@types/istanbul-reports@3.0.4': + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + + '@types/jest@29.5.14': + resolution: {integrity: sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==} + '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} @@ -2376,6 +2409,15 @@ packages: '@types/react@18.3.5': resolution: {integrity: sha512-WeqMfGJLGuLCqHGYRGHxnKrXcTitc6L/nBUWfWPcTarG3t9PsquqUMuVeXZeca+mglY4Vo5GZjCi0A3Or2lnxA==} + '@types/stack-utils@2.0.3': + resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} + + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.33': + resolution: {integrity: sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==} + '@vercel/postgres@0.10.0': resolution: {integrity: sha512-fSD23DxGND40IzSkXjcFcxr53t3Tiym59Is0jSYIFpG4/0f0KO9SGtcp1sXiebvPaGe7N/tU05cH4yt2S6/IPg==} engines: {node: '>=18.14'} @@ -2678,6 +2720,10 @@ packages: peerDependencies: devtools-protocol: '*' + ci-info@3.9.0: + resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} + engines: {node: '>=8'} + class-variance-authority@0.7.0: resolution: {integrity: sha512-jFI8IQw4hczaL4ALINxqLEXQbWcNjoSkloa4IaufXCJr6QawJyw7tuRysRsrE8w2p/4gGaxKIt/hX3qz/IbD1A==} @@ -2839,6 +2885,10 @@ packages: diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} + diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -3020,6 +3070,10 @@ packages: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} + escape-string-regexp@2.0.0: + resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} + engines: {node: '>=8'} + estree-walker@2.0.2: resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} @@ -3038,6 +3092,10 @@ packages: resolution: {integrity: sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==} engines: {node: '>=12.0.0'} + expect@29.7.0: + resolution: {integrity: sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + fast-glob@3.3.2: resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} engines: {node: '>=8.6.0'} @@ -3231,6 +3289,26 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jest-diff@29.7.0: + resolution: {integrity: sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-get-type@29.6.3: + resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-matcher-utils@29.7.0: + resolution: {integrity: sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-message-util@29.7.0: + resolution: {integrity: sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jiti@1.21.6: resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==} hasBin: true @@ -3619,6 +3697,10 @@ packages: resolution: {integrity: sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} + pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} @@ -3660,6 +3742,9 @@ packages: react-is@17.0.2: resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} + react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + react-redux@7.2.9: resolution: {integrity: sha512-Gx4L3uM182jEEayZfRbI/G11ZpYdNAnBs70lFVMNdHJI76XYtR+7m0MN+eAs7UHBPhWXcnFPaS+9owSCJQHNpQ==} peerDependencies: @@ -3823,6 +3908,10 @@ packages: simple-swizzle@0.2.2: resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} + slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + smee-client@2.0.3: resolution: {integrity: sha512-W5tQKHzZFe+IMBlaAJ8Ho32Y2wbUbzriHAA2DAFXpITId+0dYHJJbAX36a/HMrGjW7yFjhcKCNPwRBAiIrlZGQ==} hasBin: true @@ -3854,6 +3943,10 @@ packages: peerDependencies: svelte: ^4.0.0 || ^5.0.0-next.0 + stack-utils@2.0.6: + resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} + engines: {node: '>=10'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -5132,6 +5225,23 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@jest/expect-utils@29.7.0': + dependencies: + jest-get-type: 29.6.3 + + '@jest/schemas@29.6.3': + dependencies: + '@sinclair/typebox': 0.27.8 + + '@jest/types@29.6.3': + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 22.5.4 + '@types/yargs': 17.0.33 + chalk: 4.1.2 + '@jridgewell/gen-mapping@0.3.5': dependencies: '@jridgewell/set-array': 1.2.1 @@ -5892,6 +6002,8 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.22.4': optional: true + '@sinclair/typebox@0.27.8': {} + '@swc/core-darwin-arm64@1.7.42': optional: true @@ -6034,6 +6146,21 @@ snapshots: '@types/react': 18.3.5 hoist-non-react-statics: 3.3.2 + '@types/istanbul-lib-coverage@2.0.6': {} + + '@types/istanbul-lib-report@3.0.3': + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + + '@types/istanbul-reports@3.0.4': + dependencies: + '@types/istanbul-lib-report': 3.0.3 + + '@types/jest@29.5.14': + dependencies: + expect: 29.7.0 + pretty-format: 29.7.0 + '@types/js-yaml@4.0.9': {} '@types/node-fetch@2.6.11': @@ -6079,6 +6206,14 @@ snapshots: '@types/prop-types': 15.7.12 csstype: 3.1.3 + '@types/stack-utils@2.0.3': {} + + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.33': + dependencies: + '@types/yargs-parser': 21.0.3 + '@vercel/postgres@0.10.0': dependencies: '@neondatabase/serverless': 0.9.5 @@ -6439,6 +6574,8 @@ snapshots: urlpattern-polyfill: 10.0.0 zod: 3.23.8 + ci-info@3.9.0: {} + class-variance-authority@0.7.0: dependencies: clsx: 2.0.0 @@ -6590,6 +6727,8 @@ snapshots: diff-match-patch@1.0.5: {} + diff-sequences@29.6.3: {} + diff@4.0.2: {} dlv@1.1.3: {} @@ -6792,6 +6931,8 @@ snapshots: escape-string-regexp@1.0.5: {} + escape-string-regexp@2.0.0: {} + estree-walker@2.0.2: {} estree-walker@3.0.3: @@ -6804,6 +6945,14 @@ snapshots: eventsource@2.0.2: {} + expect@29.7.0: + dependencies: + '@jest/expect-utils': 29.7.0 + jest-get-type: 29.6.3 + jest-matcher-utils: 29.7.0 + jest-message-util: 29.7.0 + jest-util: 29.7.0 + fast-glob@3.3.2: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -6997,6 +7146,43 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jest-diff@29.7.0: + dependencies: + chalk: 4.1.2 + diff-sequences: 29.6.3 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-get-type@29.6.3: {} + + jest-matcher-utils@29.7.0: + dependencies: + chalk: 4.1.2 + jest-diff: 29.7.0 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-message-util@29.7.0: + dependencies: + '@babel/code-frame': 7.24.7 + '@jest/types': 29.6.3 + '@types/stack-utils': 2.0.3 + chalk: 4.1.2 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + pretty-format: 29.7.0 + slash: 3.0.0 + stack-utils: 2.0.6 + + jest-util@29.7.0: + dependencies: + '@jest/types': 29.6.3 + '@types/node': 22.5.4 + chalk: 4.1.2 + ci-info: 3.9.0 + graceful-fs: 4.2.11 + picomatch: 2.3.1 + jiti@1.21.6: {} jose@5.8.0: {} @@ -7343,6 +7529,12 @@ snapshots: ansi-styles: 5.2.0 react-is: 17.0.2 + pretty-format@29.7.0: + dependencies: + '@jest/schemas': 29.6.3 + ansi-styles: 5.2.0 + react-is: 18.3.1 + prop-types@15.8.1: dependencies: loose-envify: 1.4.0 @@ -7395,6 +7587,8 @@ snapshots: react-is@17.0.2: {} + react-is@18.3.1: {} + react-redux@7.2.9(react-dom@19.0.0-rc-7771d3a7-20240827(react@19.0.0-rc-7771d3a7-20240827))(react@19.0.0-rc-7771d3a7-20240827): dependencies: '@babel/runtime': 7.25.6 @@ -7588,6 +7782,8 @@ snapshots: is-arrayish: 0.3.2 optional: true + slash@3.0.0: {} + smee-client@2.0.3: dependencies: commander: 12.1.0 @@ -7621,6 +7817,10 @@ snapshots: svelte: 4.2.19 swrev: 4.0.0 + stack-utils@2.0.6: + dependencies: + escape-string-regexp: 2.0.0 + stackback@0.0.2: {} std-env@3.7.0: {} From b4315c44f20cb06e8166fa36c7ad7f6e87ae28cb Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 01:40:25 -0800 Subject: [PATCH 06/12] feat: add retry logic and improve error handling in AI client --- packages/shortest/src/ai/ai.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/shortest/src/ai/ai.ts b/packages/shortest/src/ai/ai.ts index ee5f3edc..5030e5dc 100644 --- a/packages/shortest/src/ai/ai.ts +++ b/packages/shortest/src/ai/ai.ts @@ -21,6 +21,28 @@ export class AIClient { browserTool: BrowserTool, outputCallback?: (content: Anthropic.Beta.Messages.BetaContentBlockParam) => void, toolOutputCallback?: (name: string, input: any) => void + ) { + const maxRetries = 3; + let attempts = 0; + + while (attempts < maxRetries) { + try { + return await this.makeRequest(prompt, browserTool, outputCallback, toolOutputCallback); + } catch (error: any) { + attempts++; + if (attempts === maxRetries) throw error; + + console.log(`Retry attempt ${attempts}/${maxRetries}`); + await new Promise(r => setTimeout(r, 5000 * attempts)); + } + } + } + + async makeRequest( + prompt: string, + browserTool: BrowserTool, + outputCallback?: (content: Anthropic.Beta.Messages.BetaContentBlockParam) => void, + toolOutputCallback?: (name: string, input: any) => void ) { const messages: Anthropic.Beta.Messages.BetaMessageParam[] = []; From 5a59551ccd5c410b46bf85e90961b6b0061ce642 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 02:04:12 -0800 Subject: [PATCH 07/12] feat: add db query and error handling in login test --- app/__tests__/login.test.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/__tests__/login.test.ts b/app/__tests__/login.test.ts index 92ef82d9..69d7d6ac 100644 --- a/app/__tests__/login.test.ts +++ b/app/__tests__/login.test.ts @@ -1,5 +1,5 @@ -import { afterAll, beforeAll, define, UITestBuilder, expect } from 'shortest'; -import { client, db } from "@/lib/db/drizzle"; +import { define, UITestBuilder, expect } from 'shortest'; +import { db } from "@/lib/db/drizzle"; interface LoginState { username: string; @@ -16,8 +16,22 @@ define('Validate login feature implemented with Clerk', async () => { expect(process.env.GITHUB_USERNAME).toBeDefined(); }) .when('Logged in', async () => { - const user = await db.query.users.findFirst(); - expect(user).toBeDefined(); + try { + const user = await db.query.users.findFirst({ + where: (users, { eq }) => eq(users.clerkId, process.env.GITHUB_USERNAME || '') + }); + + if (user) { + expect(user).toBeDefined(); + console.log("Found user:", user); + } else { + console.log("No user found in database - this is expected for first login"); + } + } catch (error: unknown) { + if (error instanceof Error) { + console.log("Database error", error.message); + } + } }) .expect('should redirect to /dashboard'); }); \ No newline at end of file From 47d7c06bec30b39e69db9a3431d9580da243e883 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 02:04:16 -0800 Subject: [PATCH 08/12] feat: add assertion error types and reporting --- packages/shortest/src/core/reporter.ts | 19 ++++++++++++++++++- packages/shortest/src/types/index.ts | 9 +++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/shortest/src/core/reporter.ts b/packages/shortest/src/core/reporter.ts index 3ed1cfb5..414126de 100644 --- a/packages/shortest/src/core/reporter.ts +++ b/packages/shortest/src/core/reporter.ts @@ -1,5 +1,5 @@ import pc from 'picocolors'; -import { ParsedTestSuite } from '../types'; +import { ParsedTestSuite, AssertionError } from '../types'; export type TestStatus = 'pending' | 'running' | 'passed' | 'failed'; @@ -103,4 +103,21 @@ export class Reporter { reportError(context: string, message: string) { console.error(pc.red(`\n${context} Error: ${message}`)); } + + reportAssertion( + step: string, + status: 'passed' | 'failed', + error?: AssertionError + ): void { + const icon = status === 'passed' ? 'โœ“' : 'โœ—'; + const color = status === 'passed' ? 'green' : 'red'; + + console.log(pc[color](`${icon} ${step}`)); + + if (error && status === 'failed') { + console.log(pc.red(` Expected: ${error.matcherResult?.expected}`)); + console.log(pc.red(` Received: ${error.matcherResult?.actual}`)); + console.log(pc.red(` Message: ${error.message}`)); + } + } } diff --git a/packages/shortest/src/types/index.ts b/packages/shortest/src/types/index.ts index db49658a..222fa95b 100644 --- a/packages/shortest/src/types/index.ts +++ b/packages/shortest/src/types/index.ts @@ -75,4 +75,13 @@ export interface ParsedTest { export interface ParsedTestSuite { name: string; tests: ParsedTest[]; +} + +export interface AssertionError extends Error { + matcherResult?: { + message: string; + pass: boolean; + actual: any; + expected: any; + }; } \ No newline at end of file From 508adc6769b3de4996def6a2b2131828224a1c44 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 02:04:19 -0800 Subject: [PATCH 09/12] fix: improve assertion handling and error propagation in test builder --- packages/shortest/src/ui-test-builder.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/shortest/src/ui-test-builder.ts b/packages/shortest/src/ui-test-builder.ts index 215e0e1c..f2e4924c 100644 --- a/packages/shortest/src/ui-test-builder.ts +++ b/packages/shortest/src/ui-test-builder.ts @@ -82,13 +82,25 @@ export class UITestBuilder implements UITestBuilderInterface { return this; } - private addStep( + private async addStep( type: TestStep['type'], action: string, payload?: any, assert?: () => Promise - ): void { + ): Promise { + // Store step this.steps.push({ type, action, payload, assert }); + + // Execute assertion immediately if present + if (assert) { + try { + // Just execute the function, don't wrap or modify it + await assert(); + } catch (error: any) { + // Just rethrow the original error + throw error; + } + } } } From 9b1f53119fe613b4b7bafa40845d0acdf5047b8a Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Thu, 21 Nov 2024 11:57:45 -0800 Subject: [PATCH 10/12] Update Github login flow setup --- README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ce591f5d..50efe50d 100644 --- a/README.md +++ b/README.md @@ -161,20 +161,19 @@ or use the `pnpm` command: pnpm test ``` -## Test Github MFA Login Flow with Shortest -In order to test Github MFA login in browser you need to add register shortest as OTP provider and add the OTP secret to your `.env.local` file: - -1. Go to your repo settings -2. Navigate to `Password and Authentication` -3. Click on `Authenticator App` -4. Choose `Use your authenticator app` -5. Click on `Setup key` to grab the OTP secret -6. Add the OTP secret to your `.env.local` file or use the `shortest` cli to add it: - +## Testing Github MFA Login Flow with Shortest +To test Github MFA login in a browser, you need to register Shortest as an OTP provider and add the OTP secret to your `.env.local` file: + +1. Go to your repository settings +2. Navigate to "Password and Authentication" +3. Click on "Authenticator App" +4. Select "Use your authenticator app" +5. Click "Setup key" to obtain the OTP secret +6. Add the OTP secret to your `.env.local` file or use the Shortest CLI to add it +7. Enter the 2FA code displayed in your terminal into Github's Authenticator setup page to complete the process ```bash shortest --github-code --secret= ``` -7. Laslty enter the 2FA code generated in terminal in your Github Authenticator editor ## To run tests in Github workflows From 8987047a90475b4ccbd657f46979f25494290fc7 Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Fri, 22 Nov 2024 00:28:12 -0800 Subject: [PATCH 11/12] Delete db validation --- app/__tests__/login.test.ts | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/app/__tests__/login.test.ts b/app/__tests__/login.test.ts index 69d7d6ac..c706c527 100644 --- a/app/__tests__/login.test.ts +++ b/app/__tests__/login.test.ts @@ -1,5 +1,4 @@ import { define, UITestBuilder, expect } from 'shortest'; -import { db } from "@/lib/db/drizzle"; interface LoginState { username: string; @@ -12,26 +11,7 @@ define('Validate login feature implemented with Clerk', async () => { .given('Github username and password', { username: process.env.GITHUB_USERNAME || '', password: process.env.GITHUB_PASSWORD || '' - }, async () => { - expect(process.env.GITHUB_USERNAME).toBeDefined(); - }) - .when('Logged in', async () => { - try { - const user = await db.query.users.findFirst({ - where: (users, { eq }) => eq(users.clerkId, process.env.GITHUB_USERNAME || '') - }); - - if (user) { - expect(user).toBeDefined(); - console.log("Found user:", user); - } else { - console.log("No user found in database - this is expected for first login"); - } - } catch (error: unknown) { - if (error instanceof Error) { - console.log("Database error", error.message); - } - } }) + .when('Logged in') .expect('should redirect to /dashboard'); }); \ No newline at end of file From 56d0f07430145d1f8d6b5b180a9cf1bb2e2e043e Mon Sep 17 00:00:00 2001 From: Mohammad Rad Date: Fri, 22 Nov 2024 00:58:51 -0800 Subject: [PATCH 12/12] delete console log --- packages/shortest/src/core/parser.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/shortest/src/core/parser.ts b/packages/shortest/src/core/parser.ts index 5a07cf26..9b6b328c 100644 --- a/packages/shortest/src/core/parser.ts +++ b/packages/shortest/src/core/parser.ts @@ -25,7 +25,6 @@ export class TestParser { tests: builders.map((builder: UITestBuilderInterface) => this.parseTestBuilder(builder)) }; - console.log(`Test Suite: ${suiteName}`); suite.tests.forEach(test => this.generateTestPrompt(test, suiteName)); this.processedSuites.add(suiteName);