feat(chrome-devtool): allow tracking active tab in bridge mode (#282)

---------

Co-authored-by: zhouxiao.shaw <zhouxiao.shaw@bytedance.com>

Author: yuyutaotao

apps/site/docs/en/automate-with-scripts-in-yaml.mdx

@@ -34,13 +34,13 @@ Config the OpenAI API key in the environment variable
 export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
 ```
 
-or you can use a `.env` file to store the configuration
+or you can use a `.env` file to store the configuration, Midscene command line tool will automatically load it when running yaml scripts.
 
 ```env filename=.env
 OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
 ```
 
-or you may [customize model and provider](./model-provider)
+For more details about model and provider, see [customize model and provider](./model-provider)
 
 ## Start
 

apps/site/docs/en/bridge-mode-by-chrome-extension.mdx

@@ -79,11 +79,13 @@ You should always call `connectCurrentTab` or `connectNewTabWithUrl` before doin
 Each of the agent instance can only connect to one tab instance, and it cannot be reconnected after destroy.
 :::
 
-### `connectCurrentTab`
+### `connectCurrentTab(options?: { trackingActiveTab?: boolean })`
 
 Connect to the current active tab on Chrome.
 
-### `connectNewTabWithUrl(ur: string)`
+If `trackingActiveTab` is true, the agent will always track the active tab. For example, if you switch to another tab or a new tab is opened, the agent will track the latest active tab. Otherwise, the agent will only track the tab you connected to initially.
+
+### `connectNewTabWithUrl(url: string, options?: { trackingActiveTab?: boolean })`
 
 Create a new tab with url and connect to immediately.
 

apps/site/docs/zh/automate-with-scripts-in-yaml.mdx

@@ -34,13 +34,13 @@ tasks:
 export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
 ```
 
-或使用 `.env` 文件存储配置
+或使用 `.env` 文件存储配置，Midscene 命令行工具在运行 yaml 脚本时会自动加载它
 
 ```env filename=.env
 OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
 ```
 
-或 [自定义模型和服务商](./model-provider)
+更多关于模型和服务商的配置，请参阅 [自定义模型和服务商](./model-provider)
 
 ## 开始
 

apps/site/docs/zh/bridge-mode-by-chrome-extension.mdx

@@ -79,11 +79,13 @@ tsx demo-new-tab.ts
 每个 agent 实例只能连接到一个标签页实例，并且一旦被销毁，就无法重新连接。
 :::
 
-### `connectCurrentTab`
+### `connectCurrentTab(options?: { trackingActiveTab?: boolean })`
 
 连接到当前已激活的标签页。
 
-### `connectNewTabWithUrl(ur: string)`
+如果 `trackingActiveTab` 为 true，则 agent 将始终跟踪当前激活的标签页。例如，如果你切换到另一个标签页或打开一个新的标签页，agent 将跟踪最新激活的标签页。否则，agent 将只跟踪你最初连接的标签页。
+
+### `connectNewTabWithUrl(url: string, options?: { trackingActiveTab?: boolean })`
 
 创建一个新标签页，并立即连接到它。
 

packages/web-integration/src/bridge-mode/agent-cli-side.ts

@@ -1,8 +1,8 @@
 import assert from 'node:assert';
 import { PageAgent } from '@/common/agent';
-import { paramStr, typeStr } from '@/common/ui-utils';
 import type { KeyboardAction, MouseAction } from '@/page';
 import {
+  type BridgeConnectTabOptions,
   BridgeEvent,
   BridgePageType,
   DefaultBridgeServerPort,
@@ -101,13 +101,13 @@ export class AgentOverChromeBridge extends PageAgent<ChromeExtensionPageCliSide>
     });
   }
 
-  async connectNewTabWithUrl(url: string) {
-    await this.page.connectNewTabWithUrl(url);
+  async connectNewTabWithUrl(url: string, options?: BridgeConnectTabOptions) {
+    await this.page.connectNewTabWithUrl(url, options);
     await sleep(500);
   }
 
-  async connectCurrentTab() {
-    await this.page.connectCurrentTab();
+  async connectCurrentTab(options?: BridgeConnectTabOptions) {
+    await this.page.connectCurrentTab(options);
     await sleep(500);
   }
 

packages/web-integration/src/bridge-mode/common.ts

@@ -13,6 +13,14 @@ export enum BridgeEvent {
   ConnectCurrentTab = 'connectCurrentTab',
 }
 
+export interface BridgeConnectTabOptions {
+  /**
+   * If true, the page will always track the active tab.
+   * @default false
+   */
+  trackingActiveTab?: boolean;
+}
+
 export enum MouseEvent {
   PREFIX = 'mouse.',
   Click = 'mouse.click',

packages/web-integration/src/bridge-mode/page-browser-side.ts

@@ -2,6 +2,7 @@ import assert from 'node:assert';
 import type { KeyboardAction, MouseAction } from '@/page';
 import ChromeExtensionProxyPage from '../chrome-extension/page';
 import {
+  type BridgeConnectTabOptions,
   BridgeEvent,
   DefaultBridgeServerPort,
   KeyboardEvent,
@@ -94,22 +95,33 @@ export class ChromeExtensionPageBrowserSide extends ChromeExtensionProxyPage {
     return await this.setupBridgeClient();
   }
 
-  public async connectNewTabWithUrl(url: string) {
+  public async connectNewTabWithUrl(
+    url: string,
+    options?: BridgeConnectTabOptions,
+  ) {
     const tab = await chrome.tabs.create({ url });
     const tabId = tab.id;
     assert(tabId, 'failed to get tabId after creating a new tab');
 
     // new tab
     this.onLogMessage(`Creating new tab: ${url}`, 'log');
+
+    if (options?.trackingActiveTab) {
+      this.trackingActiveTab = true;
+    }
   }
 
-  public async connectCurrentTab() {
+  public async connectCurrentTab(options?: BridgeConnectTabOptions) {
     const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
     console.log('current tab', tabs);
     const tabId = tabs[0]?.id;
     assert(tabId, 'failed to get tabId');
 
     this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, 'log');
+
+    if (options?.trackingActiveTab) {
+      this.trackingActiveTab = true;
+    }
   }
 
   async destroy() {

packages/web-integration/tests/ai/bridge/agent.test.ts

@@ -2,8 +2,11 @@ import {
   AgentOverChromeBridge,
   getBridgePageInCliSide,
 } from '@/bridge-mode/agent-cli-side';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 
+vi.setConfig({
+  testTimeout: 60 * 1000,
+});
 const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 describe.skipIf(process.env.CI)(
   'fully functional agent in server(cli) side',
@@ -16,54 +19,51 @@ describe.skipIf(process.env.CI)(
       await page.destroy();
     });
 
-    it(
-      'page in cli side',
-      async () => {
-        const page = getBridgePageInCliSide();
+    it('page in cli side', async () => {
+      const page = getBridgePageInCliSide();
+
+      // make sure the extension bridge is launched before timeout
+      await page.connectNewTabWithUrl('https://www.baidu.com');
 
-        // make sure the extension bridge is launched before timeout
-        await page.connectNewTabWithUrl('https://www.baidu.com');
+      // sleep 3s
+      await sleep(3000);
+
+      await page.destroy();
+    });
 
-        // sleep 3s
-        await sleep(3000);
+    it('agent in cli side, new tab', async () => {
+      const agent = new AgentOverChromeBridge();
 
-        await page.destroy();
-      },
-      40 * 1000, // longer than the timeout of the bridge io
-    );
+      await agent.connectNewTabWithUrl('https://www.bing.com');
+      await sleep(3000);
 
-    it(
-      'agent in cli side, new tab',
-      async () => {
-        const agent = new AgentOverChromeBridge();
+      await agent.ai('type "AI 101" and hit Enter');
+      await sleep(3000);
 
-        await agent.connectNewTabWithUrl('https://www.bing.com');
-        await sleep(3000);
+      await agent.aiAssert('there are some search results');
+      await agent.destroy();
+    });
 
-        await agent.ai('type "AI 101" and hit Enter');
-        await sleep(3000);
+    it('agent in cli side, current tab', async () => {
+      const agent = new AgentOverChromeBridge();
+      await agent.connectCurrentTab();
+      await sleep(3000);
+      const answer = await agent.aiQuery(
+        'name of the current page? return {name: string}',
+      );
 
-        await agent.aiAssert('there are some search results');
-        await agent.destroy();
-      },
-      60 * 1000,
-    );
+      console.log(answer);
+      expect(answer.name).toBeTruthy();
+      await agent.destroy();
+    });
 
-    it(
-      'agent in cli side, current tab',
-      async () => {
-        const agent = new AgentOverChromeBridge();
-        await agent.connectCurrentTab();
-        await sleep(3000);
-        const answer = await agent.aiQuery(
-          'name of the current page? return {name: string}',
-        );
+    it('agent in cli side, current tab, tracking active tab', async () => {
+      const agent = new AgentOverChromeBridge();
+      await agent.connectCurrentTab({ trackingActiveTab: true });
 
-        console.log(answer);
-        expect(answer.name).toBeTruthy();
-        await agent.destroy();
-      },
-      60 * 1000,
-    );
+      await agent.ai('click "文库"，sleep 1500ms，type "AI 101" and hit Enter');
+      await sleep(3000);
+      await agent.destroy();
+    });
   },
 );