Feat: Github Login Flow with MFA (#84)

Issue #69: This PR adds a feature to test Login functionality with Github OAuth. Devs need to add shortest as their OTP provider in their Github settings. Then shortest can act on behalf of devs to test the MFA flow using Authenticator app. Updates: 1. Add Github OTP Code generation 2. Add Github login tool for AI to use to login to the web app 3. Add default caching mechanism to save the users session on the following test cases 4. Add clear cache tool for AI to use to reset user session. (AI will use this if users are specifically testing a login flow" 5. Add new shortest flag to add TOTP secret and generate TOTP code manually if needed: `shortest --github-code --secret=<TOTP_SECRET>` 6. Update setup script to add TOTP secret to .env
anti-work · Nov 21, 2024 · f164c1b · f164c1b
2 parents 3d22480 + 0cae957
commit f164c1b
Show file tree

Hide file tree

Showing 20 changed files with 716 additions and 206 deletions.
diff --git a/.env.example b/.env.example
@@ -26,4 +26,9 @@ POSTGRES_PASSWORD=
 POSTGRES_DATABASE=
 
 # Copy from Anthropic Dashboard Settings
-ANTHROPIC_API_KEY=
+ANTHROPIC_API_KEY=
+
+# Copy from Github Settings
+GITHUB_TOTP_SECRET=
+GITHUB_USERNAME=
+GITHUB_PASSWORD=
diff --git a/.gitignore b/.gitignore
@@ -48,4 +48,7 @@ docker-compose.override.yml
 docker-compose.yml
 
 # screenshots
-screenshots/
+screenshots/
+
+# browser data
+.browser-data/
diff --git a/README.md b/README.md
@@ -161,6 +161,22 @@ or use the `pnpm` command:
 pnpm test
 ```
 
+## Test Github MFA Login Flow with Shortest
+In order to test Github MFA login in browser you need to add register shortest as OTP provider and add the OTP secret to your `.env.local` file:
+
+1. Go to your repo settings
+2. Navigate to `Password and Authentication`
+3. Click on `Authenticator App`
+4. Choose `Use your authenticator app`
+5. Click on `Setup key` to grab the OTP secret
+6. Add the OTP secret to your `.env.local` file or use the `shortest` cli to add it:
+
+```bash
+shortest --github-code --secret=<OTP_SECRET>
+```
+7. Laslty enter the 2FA code generated in terminal in your Github Authenticator editor
+
+
 ## To run tests in Github workflows
 
 prerequisites:
@@ -174,3 +190,4 @@ prerequisites:
 - Navigate to `Security`
 - Click `Add secret`
 - Add the ANTHROPIC_API_KEY environment variable
+
diff --git a/app/__tests__/home.test.ts b/app/__tests__/home.test.ts
@@ -4,11 +4,12 @@ interface loginButton {
   url: string;
 }
 
-define('Validate login button in home page', () => {
+define('Validate Dasboard is accessible by users', () => {
 
   new UITestBuilder<loginButton>('/')
-    .test('Validate login button is visible and works')
+    .test('Validate that users can access the dashboard')
     .given('baseUrl', { url: 'http://localhost:3000' })
-    .expect('Should redirect to login page')
+    .when('Clicking on view dashboard button')
+    .expect('Should be able redirect to /dashboard and see the dashboard')
 
 });
diff --git a/app/__tests__/login.test.ts b/app/__tests__/login.test.ts
@@ -7,15 +7,15 @@ interface LoginState {
 
 define('Validate login feature implemented with Clerk', () => {
 
-  new UITestBuilder<LoginState>('/')
-    .test('Login to the app using Email and Password')
-    .given('username and password', { username: 'argo.mohrad@gmail.com', password: 'Shortest1234' })
-    .when('Logged in, click on view dashboard button')
-    .expect('should successfully redirect to /')
+  // new UITestBuilder<LoginState>('/')
+    // .test('Login to the app using Email and Password')
+    // .given('username and password', { username: 'argo.mohrad@gmail.com', password: 'Shortest1234' })
+    // .when('Logged in, click on view dashboard button')
+    // .expect('should successfully redirect to /')
 
-    // new UITestBuilder<LoginState>('/')
-    // .test('Login to the app using Google account')
-    // .given('username and password', { username: 'test', password: 'test' })
-    // .expect('should successfully redirect to /dashboard')
+    new UITestBuilder<LoginState>('/')
+    .test('Login to the app using Github login')
+    .given('Github username and password', { username: `${process.env.GITHUB_USERNAME}`, password: `${process.env.GITHUB_PASSWORD}` })
+    .expect('should successfully redirect to /dashboard')
 
 });
diff --git a/lib/setup.ts b/lib/setup.ts
@@ -253,6 +253,18 @@ async function promptForGitHubOAuth(): Promise<void> {
   }
 }
 
+async function promptForGitHubTOTP(): Promise<string | undefined> {
+  console.log('\nStep 8: GitHub 2FA TOTP Setup (Optional)');
+  console.log('If you want to test GitHub 2FA login, you\'ll need to add a TOTP secret.');
+
+  const setupNow = await question('Would you like to set up GitHub 2FA TOTP now? (y/n): ');
+
+  if (setupNow.toLowerCase() === 'y') {
+    return await question('Enter your GitHub TOTP secret (from repo settings > Password and Authentication): ');
+  }
+  return undefined;
+}
+
 async function writeEnvFile(envVars: Record<string, string>) {
   console.log('Step 8: Writing environment variables to .env.local');
   const envContent = Object.entries(envVars)
@@ -290,6 +302,7 @@ async function main() {
   const { publishableKey: NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, secretKey: CLERK_SECRET_KEY } = await promptForClerkKeys();
   const ANTHROPIC_API_KEY = await promptForAnthropicApiKey();
   await promptForGitHubOAuth();
+  const GITHUB_TOTP_SECRET = await promptForGitHubTOTP() || '';
   const CLERK_SIGN_IN_FALLBACK_REDIRECT_URL = '/dashboard';
   const CLERK_SIGN_UP_FALLBACK_REDIRECT_URL = '/dashboard';
   const NEXT_PUBLIC_CLERK_SIGN_IN_URL = '/signin';
@@ -314,6 +327,7 @@ async function main() {
     POSTGRES_PASSWORD,
     POSTGRES_DATABASE,
     ANTHROPIC_API_KEY,
+    GITHUB_TOTP_SECRET,
   });
 
   console.log('🎉 Setup completed successfully!');

diff --git a/packages/shortest/package.json b/packages/shortest/package.json
@@ -30,7 +30,8 @@
     "postinstall": "pnpm exec playwright install",
     "test:ai": "tsx scripts/test-ai.ts",
     "test:browser": "tsx scripts/test-browser.ts",
-    "test:coordinates": "tsx scripts/test-coordinates.ts"
+    "test:coordinates": "tsx scripts/test-coordinates.ts",
+    "test:github": "tsx scripts/test-github.ts"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "0.32.0",
@@ -40,6 +41,7 @@
     "esbuild": "^0.20.1",
     "glob": "^10.3.10",
     "dotenv": "^16.4.5",
+    "otplib": "^12.0.1",
     "picocolors": "^1.0.0",
     "playwright": "^1.42.1"
   },

diff --git a/packages/shortest/scripts/test-ai.ts b/packages/shortest/scripts/test-ai.ts
@@ -3,26 +3,31 @@ import { BrowserTool } from '../src/browser-use/browser';
 import { defaultConfig, initialize } from '../src/index';
 import { AIClient } from '../src/ai/ai';
 import Anthropic from '@anthropic-ai/sdk';
+import pc from 'picocolors';
 
 async function testBrowser() {
   const browserManager = new BrowserManager();
 
   const apiKey = defaultConfig.ai?.apiKey || process.env.ANTHROPIC_API_KEY;
   if (!apiKey) {
-    console.error('Error: Anthropic API key not found in config or environment');
+    console.error(pc.red('Error: Anthropic API key not found in config or environment'));
     process.exit(1);
   }
 
   try {
     await initialize();
-    console.log('🚀 Launching browser...');
+    console.log(pc.cyan('🚀 Launching browser...'));
     const context = await browserManager.launch();
     const page = context.pages()[0];
 
-    const browserTool = new BrowserTool(page, {
-      width: 1920, 
-      height: 940
-    });
+    const browserTool = new BrowserTool(
+      page,
+      browserManager,
+      {
+        width: 1920, 
+        height: 940
+      }
+    );
 
     // Initialize AI client
     const aiClient = new AIClient({
@@ -31,19 +36,25 @@ async function testBrowser() {
       maxMessages: 10
     });
 
-    // Define callbacks
+    // Define callbacks with metadata logging
     const outputCallback = (content: Anthropic.Beta.Messages.BetaContentBlockParam) => {
       if (content.type === 'text') {
-        console.log('🤖 Assistant:', content.text);
+        console.log(pc.yellow('🤖 Assistant:'), content.text);
       }
     };
 
     const toolOutputCallback = (name: string, input: any) => {
-      console.log('🔧 Tool Use:', name, input);
+      console.log(pc.yellow('🔧 Tool Use:'), name, input);
+      if (input.metadata) {
+        console.log(pc.yellow('Tool Metadata:'), input.metadata);
+      }
     };
 
     // Run test
-    const testPrompt = `Validate the sign in functionality of the website you see`;
+    const testPrompt = `Validate the login functionality of the website you see
+    using github login button
+    for username argo.mohrad@gmail.com and password: M2@rad99308475
+    `;
 
     const result = await aiClient.processAction(
       testPrompt,
@@ -52,16 +63,16 @@ async function testBrowser() {
       toolOutputCallback
     );
 
-    console.log('✅ Test complete');
+    console.log(pc.green('✅ Test complete'));
 
   } catch (error) {
-    console.error('❌ Test failed:', error);
+    console.error(pc.red('❌ Test failed:'), error);
   } finally {
-    console.log('\n🧹 Cleaning up...');
+    console.log(pc.cyan('\n🧹 Cleaning up...'));
     await browserManager.close();
   }
 }
 
-console.log('🧪 Browser Integration Test');
-console.log('===========================');
+console.log(pc.cyan('🧪 Browser Integration Test'));
+console.log(pc.cyan('==========================='));
 testBrowser().catch(console.error);
diff --git a/packages/shortest/scripts/test-browser.ts b/packages/shortest/scripts/test-browser.ts
@@ -1,20 +1,25 @@
 import { BrowserManager } from '../src/core/browser-manager';
 import { BrowserTool } from '../src/browser-use/browser';
 import { initialize } from '../src/index';
+import pc from 'picocolors';
 
 async function testBrowser() {
   const browserManager = new BrowserManager();
 
   try {
     await initialize();
-    console.log('🚀 Launching browser...');
+    console.log(pc.cyan('🚀 Launching browser...'));
     const context = await browserManager.launch();
     const page = context.pages()[0];
 
-    const browserTool = new BrowserTool(page, {
-      width: 1920, 
-      height: 1080
-    });
+    const browserTool = new BrowserTool(
+      page,
+      browserManager,
+      {
+        width: 1920, 
+        height: 1080
+      }
+    );
 
     // Navigate to a page with a sign in button
     await page.goto('http://localhost:3000');
@@ -31,48 +36,56 @@ async function testBrowser() {
     const x = Math.round(boundingBox.x + boundingBox.width / 2);
     const y = Math.round(boundingBox.y + boundingBox.height / 2);
 
-    console.log(`📍 Sign in button coordinates: (${x}, ${y})`);
+    console.log(pc.cyan(`📍 Sign in button coordinates: (${x}, ${y})`));
 
     // Test sequence
-    console.log('\n📍 Testing Mouse Movements and Clicks:');
+    console.log(pc.cyan('\n📍 Testing Mouse Movements and Clicks:'));
 
     // Move to sign in button
-    console.log(`\nTest 1: Move to Sign in button (${x}, ${y})`);
-    await browserTool.execute({ 
+    console.log(pc.cyan(`\nTest 1: Move to Sign in button (${x}, ${y})`));
+    const moveResult = await browserTool.execute({ 
       action: 'mouse_move', 
       coordinates: [x, y] 
     });
+    console.log(pc.yellow('\nMouse Move Result:'), moveResult);
+    console.log(pc.yellow('Metadata:'), moveResult.metadata);
     await new Promise(r => setTimeout(r, 1000));
 
     // Take screenshot to verify position
-    console.log('\nTest 2: Taking screenshot to verify cursor position');
-    await browserTool.execute({ 
+    console.log(pc.cyan('\nTest 2: Taking screenshot to verify cursor position'));
+    const screenshotResult = await browserTool.execute({ 
       action: 'screenshot' 
     });
+    console.log(pc.yellow('\nScreenshot Result:'), screenshotResult);
+    console.log(pc.yellow('Metadata:'), screenshotResult.metadata);
 
     // Click the button
-    console.log('\nTest 3: Clicking at current position');
-    await browserTool.execute({ 
+    console.log(pc.cyan('\nTest 3: Clicking at current position'));
+    const clickResult = await browserTool.execute({ 
       action: 'left_click'
     });
+    console.log(pc.yellow('\nClick Result:'), clickResult);
+    console.log(pc.yellow('Metadata:'), clickResult.metadata);
     await new Promise(r => setTimeout(r, 1000));
 
     // Take final screenshot
-    console.log('\nTest 4: Taking screenshot after click');
-    const result = await browserTool.execute({ 
+    console.log(pc.cyan('\nTest 4: Taking screenshot after click'));
+    const finalResult = await browserTool.execute({ 
       action: 'screenshot' 
     });
+    console.log(pc.yellow('\nFinal Screenshot Result:'), finalResult);
+    console.log(pc.yellow('Metadata:'), finalResult.metadata);
 
-    console.log('\n✅ All coordinate tests completed');
+    console.log(pc.green('\n✅ All coordinate tests completed'));
 
   } catch (error) {
-    console.error('❌ Test failed:', error);
+    console.error(pc.red('❌ Test failed:'), error);
   } finally {
-    console.log('\n🧹 Cleaning up...');
+    console.log(pc.cyan('\n🧹 Cleaning up...'));
     await browserManager.close();
   }
 }
 
-console.log('🧪 Mouse Coordinate Test');
-console.log('=======================');
+console.log(pc.cyan('🧪 Mouse Coordinate Test'));
+console.log(pc.cyan('======================='));
 testBrowser().catch(console.error);