feat(app): supports control of iOS and Android devices through appium (…

…#82)


Co-authored-by: zhouxiao.shaw <zhouxiao.shaw@bytedance.com>
Co-authored-by: linyibing <linyibing@bytedance.com>

CONTRIBUTING.md

@@ -142,9 +142,12 @@ npx nx test:all @midscene/web
 
 ### Run E2E Tests
 
-Midscene uses [playwright](https://github.com/microsoft/playwright) to run end-to-end tests.
+Midscene uses
 
-You can run the `e2e` command to run E2E tests:
+- [playwright](https://github.com/microsoft/playwright) to run end-to-end tests.
+- [appium](https://github.com/appium/appium) to run end-to-end tests on iOS/Android.
+
+You can run the `e2e` command to run E2E tests for playwright:
 
 ```sh
 pnpm run e2e
@@ -156,6 +159,13 @@ If you need to run a specified test:
 npx nx e2e @midscene/web
 ```
 
+If you need to run E2E tests for appium:
+> Before running the test, you need to start the appium server first, please refer to the [README.md](./packages/web-integration/README.md) for details.
+
+```sh
+cd packages/web-integration && pnpm run test:ai -- appium
+```
+
 ---
 
 ## Linting

packages/midscene/src/ai-model/automation/planning.ts

@@ -12,12 +12,12 @@ export function systemPromptToTaskPlanning() {
     * param: null
   * type: 'Hover', hover the previous element found
     * param: null
-  * type: 'Input', input something
-    * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original).
+  * type: 'Input', replace the value in the input field
+    * param: { value: string }, The input value must not be an empty string. Provide a meaningful final required input value based on the existing input. No matter what modifications are required, just provide the final value to replace the existing input value. After locating the input field, do not use 'Tap' action, proceed directly to 'Input' action.
   * type: 'KeyboardPress',  press a key
     * param: { value: string },  the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
   * type: 'Scroll'
-    * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
+    * param: { scrollType: 'scrollDownOneScreen', 'scrollUpOneScreen', 'scrollUntilBottom', 'scrollUntilTop' }
   * type: 'Error'
     * param: { message: string }, the error message
   * type: 'Sleep'

packages/midscene/src/types.ts

@@ -210,10 +210,10 @@ export interface PlanningActionParamInputOrKeyPress {
 }
 export interface PlanningActionParamScroll {
   scrollType:
-    | 'ScrollUntilBottom'
-    | 'ScrollUntilTop'
-    | 'ScrollDown'
-    | 'ScrollUp';
+    | 'scrollUntilTop'
+    | 'scrollUntilBottom'
+    | 'scrollUpOneScreen'
+    | 'scrollDownOneScreen';
 }
 
 export interface PlanningActionParamAssert {

packages/midscene/src/utils.ts

@@ -1,12 +1,6 @@
 import assert from 'node:assert';
 import { randomUUID } from 'node:crypto';
-import {
-  copyFileSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  writeFileSync,
-} from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { basename, dirname, join } from 'node:path';
 import type { Rect, ReportDumpWithAttributes } from './types';