From a7fffcaf7506d5e8b1f463cf2b76e8fb20e26e50 Mon Sep 17 00:00:00 2001 From: zouyukun Date: Thu, 25 Dec 2025 17:31:46 +0800 Subject: [PATCH] hdc --- phone_agent/actions/handler.py | 15 +++++++-- phone_agent/device_factory.py | 14 +++++--- phone_agent/hdc/input.py | 59 +++++++++++++++++++++++----------- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/phone_agent/actions/handler.py b/phone_agent/actions/handler.py index 0bef1c3a..1d4fbef6 100644 --- a/phone_agent/actions/handler.py +++ b/phone_agent/actions/handler.py @@ -41,6 +41,8 @@ def __init__( self.device_id = device_id self.confirmation_callback = confirmation_callback or self._default_confirmation self.takeover_callback = takeover_callback or self._default_takeover + # Track last tap coordinates for passing to type_text (used by HDC) + self._last_tap_coords: tuple[int, int] | None = None def execute( self, action: dict[str, Any], screen_width: int, screen_height: int @@ -135,6 +137,9 @@ def _handle_tap(self, action: dict, width: int, height: int) -> ActionResult: x, y = self._convert_relative_to_absolute(element, width, height) + # Store tap coordinates for potential use in subsequent Type action + self._last_tap_coords = (x, y) + # Check for sensitive operation if "message" in action: if not self.confirmation_callback(action["message"]): @@ -162,8 +167,12 @@ def _handle_type(self, action: dict, width: int, height: int) -> ActionResult: device_factory.clear_text(self.device_id) time.sleep(TIMING_CONFIG.action.text_clear_delay) - # Handle multiline text by splitting on newlines - device_factory.type_text(text, self.device_id) + # Type text with last tap coordinates if available (for HDC) + if self._last_tap_coords: + x, y = self._last_tap_coords + device_factory.type_text(text, self.device_id, x, y) + else: + device_factory.type_text(text, self.device_id) time.sleep(TIMING_CONFIG.action.text_input_delay) # Restore original keyboard @@ -396,4 +405,4 @@ def do(**kwargs) -> dict[str, Any]: def finish(**kwargs) -> dict[str, Any]: """Helper function for creating 'finish' actions.""" kwargs["_metadata"] = "finish" - return kwargs + return kwargs \ No newline at end of file diff --git a/phone_agent/device_factory.py b/phone_agent/device_factory.py index 915ff52b..81f1da1e 100644 --- a/phone_agent/device_factory.py +++ b/phone_agent/device_factory.py @@ -105,9 +105,15 @@ def launch_app( """Launch an app.""" return self.module.launch_app(app_name, device_id, delay) - def type_text(self, text: str, device_id: str | None = None): - """Type text.""" - return self.module.type_text(text, device_id) + def type_text(self, text: str, device_id: str | None = None, x: int | None = None, y: int | None = None): + """Type text with optional coordinates for HDC.""" + import inspect + sig = inspect.signature(self.module.type_text) + # Check if the module supports x, y parameters (HDC does, ADB/iOS don't) + if 'x' in sig.parameters and 'y' in sig.parameters: + return self.module.type_text(text, device_id, x, y) + else: + return self.module.type_text(text, device_id) def clear_text(self, device_id: str | None = None): """Clear text.""" @@ -164,4 +170,4 @@ def get_device_factory() -> DeviceFactory: global _device_factory if _device_factory is None: _device_factory = DeviceFactory(DeviceType.ADB) # Default to ADB - return _device_factory + return _device_factory \ No newline at end of file diff --git a/phone_agent/hdc/input.py b/phone_agent/hdc/input.py index 920cf7dd..6bf27029 100644 --- a/phone_agent/hdc/input.py +++ b/phone_agent/hdc/input.py @@ -7,23 +7,29 @@ from phone_agent.hdc.connection import _run_hdc_command -def type_text(text: str, device_id: str | None = None) -> None: +def type_text(text: str, device_id: str | None = None, x: int | None = None, y: int | None = None) -> None: """ Type text into the currently focused input field. Args: text: The text to type. Supports multi-line text with newline characters. device_id: Optional HDC device ID for multi-device setups. + x: Optional X coordinate of the input field (recommended for HarmonyOS). + y: Optional Y coordinate of the input field (recommended for HarmonyOS). Note: - HarmonyOS uses: hdc shell uitest uiInput text "文本内容" - This command works without coordinates when input field is focused. + HarmonyOS command format: + - With coordinates: hdc shell uitest uiInput inputText X Y "文本内容" + - Without coordinates: hdc shell uitest uiInput text "文本内容" (may not work reliably) For multi-line text, the function splits by newlines and sends ENTER keyEvents. ENTER key code in HarmonyOS: 2054 - Recommendation: Click on the input field first to focus it, then use this function. + Recommendation: Click on the input field first to focus it, then use this function with coordinates. """ hdc_prefix = _get_hdc_prefix(device_id) + # Determine which command to use + use_input_text = x is not None and y is not None + # Handle multi-line text by splitting on newlines if '\n' in text: lines = text.split('\n') @@ -32,11 +38,18 @@ def type_text(text: str, device_id: str | None = None) -> None: # Escape special characters for shell escaped_line = line.replace('"', '\\"').replace("$", "\\$") - _run_hdc_command( - hdc_prefix + ["shell", "uitest", "uiInput", "text", escaped_line], - capture_output=True, - text=True, - ) + if use_input_text: + _run_hdc_command( + hdc_prefix + ["shell", "uitest", "uiInput", "inputText", str(x), str(y), escaped_line], + capture_output=True, + text=True, + ) + else: + _run_hdc_command( + hdc_prefix + ["shell", "uitest", "uiInput", "text", escaped_line], + capture_output=True, + text=True, + ) # Send ENTER key event after each line except the last one if i < len(lines) - 1: @@ -49,18 +62,26 @@ def type_text(text: str, device_id: str | None = None) -> None: except Exception as e: print(f"[HDC] ENTER keyEvent failed: {e}") else: - # Single line text - original logic + # Single line text # Escape special characters for shell (keep quotes for proper text handling) - # The text will be wrapped in quotes in the command escaped_text = text.replace('"', '\\"').replace("$", "\\$") - # HarmonyOS uitest uiInput text command - # Format: hdc shell uitest uiInput text "文本内容" - _run_hdc_command( - hdc_prefix + ["shell", "uitest", "uiInput", "text", escaped_text], - capture_output=True, - text=True, - ) + if use_input_text: + # HarmonyOS uitest uiInput inputText command with coordinates + # Format: hdc shell uitest uiInput inputText X Y "文本内容" + _run_hdc_command( + hdc_prefix + ["shell", "uitest", "uiInput", "inputText", str(x), str(y), escaped_text], + capture_output=True, + text=True, + ) + else: + # HarmonyOS uitest uiInput text command (fallback) + # Format: hdc shell uitest uiInput text "文本内容" + _run_hdc_command( + hdc_prefix + ["shell", "uitest", "uiInput", "text", escaped_text], + capture_output=True, + text=True, + ) def clear_text(device_id: str | None = None) -> None: @@ -146,4 +167,4 @@ def _get_hdc_prefix(device_id: str | None) -> list: """Get HDC command prefix with optional device specifier.""" if device_id: return ["hdc", "-t", device_id] - return ["hdc"] + return ["hdc"] \ No newline at end of file