diff --git a/packages/midscene/src/ai-model/prompt/util.ts b/packages/midscene/src/ai-model/prompt/util.ts index fb28cda10..72d6b383d 100644 --- a/packages/midscene/src/ai-model/prompt/util.ts +++ b/packages/midscene/src/ai-model/prompt/util.ts @@ -322,7 +322,7 @@ export async function descriptionOfTree< const indentStr = ' '.repeat(indent); let children = ''; - for (let i = 0; i < node.children.length; i++) { + for (let i = 0; i < (node.children || []).length; i++) { const childContent = buildContentTree(node.children[i], indent + 1); if (childContent) { children += `\n${childContent}`; @@ -405,11 +405,11 @@ export async function describeUserPage< const idElementMap: Record = {}; const flatElements: ElementType[] = []; function dfsTree(node: ElementTreeNode) { - if (node.node) { + if (node?.node) { idElementMap[node.node.id] = node.node; flatElements.push(node.node); } - for (let i = 0; i < node.children.length; i++) { + for (let i = 0; i < (node.children || []).length; i++) { dfsTree(node.children[i]); } } diff --git a/packages/midscene/tests/ai/connectivity.test.ts b/packages/midscene/tests/ai/connectivity.test.ts index 4812c1bff..b005bc7a1 100644 --- a/packages/midscene/tests/ai/connectivity.test.ts +++ b/packages/midscene/tests/ai/connectivity.test.ts @@ -7,6 +7,11 @@ import dotenv from 'dotenv'; import { getFixture } from 'tests/utils'; import { beforeAll, describe, expect, it, vi } from 'vitest'; +dotenv.config({ + debug: true, + override: true, +}); + vi.setConfig({ testTimeout: 20 * 1000, }); diff --git a/packages/midscene/tests/ai/evaluate/assertion.test.ts b/packages/midscene/tests/ai/evaluate/assertion.test.ts index 4930b73fb..8b6743abd 100644 --- a/packages/midscene/tests/ai/evaluate/assertion.test.ts +++ b/packages/midscene/tests/ai/evaluate/assertion.test.ts @@ -9,7 +9,12 @@ import { repeatFile, } from './test-suite/util'; import 'dotenv/config'; -import { repeatTime } from '../util'; +import dotenv from 'dotenv'; + +dotenv.config({ + debug: true, + override: true, +}); const testSources = [ 'online_order', diff --git a/packages/midscene/tests/ai/evaluate/computer.test.ts b/packages/midscene/tests/ai/evaluate/computer.test.ts deleted file mode 100644 index 78506d808..000000000 --- a/packages/midscene/tests/ai/evaluate/computer.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { callToGetJSONObject } from '@/ai-model'; -import { AIActionType } from '@/ai-model/common'; -import { compositeElementInfoImg, saveBase64Image } from '@midscene/shared/img'; -import { afterAll, describe, expect, it } from 'vitest'; -import { type InspectAiTestCase, getPageTestData } from './test-suite/util'; - -const testSources = [ - 'todo', - 'online_order', - 'online_order_list', - 'taobao', - 'aweme_login', - 'aweme_play', -]; - -const allResults: Array<{ - name: string; - successRate: string; - successCount: number; - totalCount: number; -}> = []; -describe( - 'automation - computer', - () => { - afterAll(() => { - console.table(allResults); - }); - it('basic run', async () => { - for (const source of testSources) { - const result: Array<{ - index: number; - expectation: any; - reality: string; - rectInBox: boolean; - }> = []; - const aiDataPath = path.join( - __dirname, - `ai-data/inspect/${source}.json`, - ); - const aiData = JSON.parse( - readFileSync(aiDataPath, 'utf-8'), - ) as InspectAiTestCase; - const { context } = await getPageTestData( - path.join(__dirname, aiData.testDataPath), - ); - console.log('context.size', context.size); - const promises = aiData.testCases.map(async (testCase, index) => { - const res = await callToGetJSONObject<{ x: number; y: number }>( - [ - { - role: 'system', - content: ` - take a screenshot and carefully evaluate the coordinates if you have achieved the right outcome. Explicitly show your thinking: "I have evaluated step X..." If not correct, try again. Only when you confirm a step was executed correctly should you move on to the next one. - - `, - }, - { - role: 'user', - content: [ - { - type: 'image_url', - image_url: { - url: context.originalScreenshotBase64, - }, - }, - { - type: 'text', - text: ` - pageDescription: { - display_width_px: ${context.size.width}, - display_height_px: ${context.size.height}, - } \n - - Here is the item user want to find. Just go ahead: - ===================================== - 找到（${testCase.prompt}位置），x/y坐标(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates，并按照下面的格式返回： - - { - "x": number, - "y": number - } - ===================================== - `, - }, - ], - }, - ], - AIActionType.ASSERT, - ); - const rect: any = context.content.find( - (item: any) => testCase.response[0].indexId === item.indexId, - )?.rect; - result.push({ - index, - expectation: { - prompt: testCase.prompt, - rect, - }, - reality: JSON.stringify(res), - rectInBox: - res && - typeof res === 'object' && - 'x' in res && - 'y' in res && - res.x >= rect?.left && - res.x <= rect?.left + rect?.width && - res.y >= rect?.top && - res.y <= rect?.top + rect?.height, - }); - }); - await Promise.all(promises); - // Write result to file - const resultFilePath = path.join( - __dirname, - '__ai_responses__', - 'computer', - `${source}-computer-result.json`, - ); - const composeImgPath = path.join( - __dirname, - '__ai_responses__', - 'computer', - `${source}-img.png`, - ); - const resultDir = path.dirname(resultFilePath); - - if (!existsSync(resultDir)) { - mkdirSync(resultDir, { recursive: true }); - } - - const newImg = await compositeElementInfoImg({ - inputImgBase64: context.originalScreenshotBase64, - elementsPositionInfo: result.map((item) => { - const position = JSON.parse(item.reality); - return { - indexId: item.index, - rect: { - left: position.x - 10, - top: position.y - 10, - width: 20, - height: 20, - }, - }; - }), - }); - await saveBase64Image({ - base64Data: newImg, - outputPath: composeImgPath, - }); - writeFileSync(resultFilePath, JSON.stringify(result, null, 2), 'utf-8'); - - const totalCount = result.length; - const successCount = result.filter((r) => r.rectInBox).length; - const successRate = ((successCount / totalCount) * 100).toFixed(2); - allResults.push({ - name: source, - successRate: `${successRate}%`, - successCount, - totalCount, - }); - } - }); - }, - { - timeout: 100000, - }, -); diff --git a/packages/midscene/tests/ai/evaluate/inspect.test.ts b/packages/midscene/tests/ai/evaluate/inspect.test.ts index c1ce48ddd..149038809 100644 --- a/packages/midscene/tests/ai/evaluate/inspect.test.ts +++ b/packages/midscene/tests/ai/evaluate/inspect.test.ts @@ -14,8 +14,14 @@ import { runTestCases, } from './test-suite/util'; import 'dotenv/config'; +import dotenv from 'dotenv'; import { repeatTime } from '../util'; +dotenv.config({ + debug: true, + override: true, +}); + const relocateAfterPlanning = false; const failCaseThreshold = process.env.CI ? 1 : 0; const testSources = [ diff --git a/packages/midscene/vitest.config.ts b/packages/midscene/vitest.config.ts index 1ba8b8c2c..2f7f604a8 100644 --- a/packages/midscene/vitest.config.ts +++ b/packages/midscene/vitest.config.ts @@ -10,6 +10,8 @@ import { version } from './package.json'; */ dotenv.config({ path: path.join(__dirname, '../../.env'), + override: true, + debug: true, }); const enableAiTest = Boolean(process.env.AITEST); diff --git a/packages/web-integration/src/playground/static-page.ts b/packages/web-integration/src/playground/static-page.ts index 6bdbba465..789ece66e 100644 --- a/packages/web-integration/src/playground/static-page.ts +++ b/packages/web-integration/src/playground/static-page.ts @@ -17,7 +17,20 @@ export default class StaticPage implements AbstractPage { private uiContext: WebUIContext; constructor(uiContext: WebUIContext) { - this.uiContext = uiContext; + if (uiContext.tree) { + this.uiContext = uiContext; + } else { + const contents = uiContext.content; + this.uiContext = Object.assign(uiContext, { + tree: { + node: null, + children: contents.map((content) => ({ + node: content, + children: [], + })), + }, + }); + } } async getElementsInfo() {