fix(extract-data): resolve match multiple element (#280)

Author: zhoushaw

packages/midscene/src/ai-model/prompt/ui-tars-planning.ts

@@ -58,7 +58,7 @@ export function parseActionFromVlm(
   if (mode === 'bc') {
     // Parse thought/reflection based on different text patterns
     if (text.startsWith('Thought:')) {
-      const thoughtMatch = text.match(/Thought: (.+?)(?=\s*Action:|$)/);
+      const thoughtMatch = text.match(/Thought: (.+?)(?=\s*Action:|$)/s);
       if (thoughtMatch) {
         thought = thoughtMatch[1].trim();
       }

packages/midscene/src/ai-model/prompt/util.ts

@@ -209,15 +209,25 @@ export function elementByPositionWithElementInfo(
   },
 ) {
   assert(typeof position !== 'undefined', 'position is required for query');
-  const item = elementsInfo.find((item) => {
+  const matchingElements = elementsInfo.filter((item) => {
     return (
       item.rect.left <= position.x &&
       position.x <= item.rect.left + item.rect.width &&
       item.rect.top <= position.y &&
       position.y <= item.rect.top + item.rect.height
     );
   });
-  return item;
+
+  if (matchingElements.length === 0) {
+    return undefined;
+  }
+
+  // Find the smallest element by area
+  return matchingElements.reduce((smallest, current) => {
+    const smallestArea = smallest.rect.width * smallest.rect.height;
+    const currentArea = current.rect.width * current.rect.height;
+    return currentArea < smallestArea ? current : smallest;
+  });
 }
 
 export const samplePageDescription = `

packages/midscene/tests/ai/parse-action.test.ts

@@ -7,7 +7,7 @@ import { describe, expect, it } from 'vitest';
 describe('parse action from vlm', () => {
   it('should parse action with Thought format', () => {
     const text = `Thought: 点击登录按钮
-Action: click(start_box='(200,300,400,500)')`;
+  Action: click(start_box='(200,300,400,500)')`;
 
     const actions = parseActionFromVlm(text);
     expect(actions).toHaveLength(1);
@@ -21,6 +21,25 @@ Action: click(start_box='(200,300,400,500)')`;
     });
   });
 
+  it('should parse action with Thought format2', () => {
+    const text = `Thought: To proceed with the task of opening Twitter and posting a tweet, I need to first access the Google search page. The highlighted "Google 搜索" button is the appropriate element to interact with, as it will allow me to search for Twitter and navigate to its website.
+   Click on the "Google 搜索" button to initiate a search for Twitter.
+Action: click(start_box='(460,452)')`;
+
+    const actions = parseActionFromVlm(text);
+    expect(actions).toHaveLength(1);
+    expect(actions[0]).toEqual({
+      reflection: null,
+      thought:
+        'To proceed with the task of opening Twitter and posting a tweet, I need to first access the Google search page. The highlighted "Google 搜索" button is the appropriate element to interact with, as it will allow me to search for Twitter and navigate to its website.\n   Click on the "Google 搜索" button to initiate a search for Twitter.',
+      action_type: 'click',
+      action_inputs: {
+        start_box: '[0.46,0.452,0.46,0.452]',
+      },
+    });
+  });
+  //
+
   it('should parse action with Reflection format', () => {
     const text = `Action_Summary: 输入用户名
 Action: type(content='username')`;

packages/midscene/tsconfig.json

@@ -16,7 +16,8 @@
     "rootDir": ".",
     "skipLibCheck": true,
     "strict": true,
-    "module": "ESNext"
+    "module": "ESNext",
+    "target": "es2018"
   },
   "exclude": ["**/node_modules"],
   "include": ["src", "tests", "report"]