From 759ca156699828d00a8aceeaf7cb32eb201aba3c Mon Sep 17 00:00:00 2001 From: Repobor Date: Mon, 29 Dec 2025 22:16:05 +0800 Subject: [PATCH 1/6] fix(accessibility): Fix mode switching and task initialization for accessibility mode **Problems Fixed:** 1. ComponentManager.reinitializeAgent() did not handle accessibility mode switching 2. initializeAccessibilityComponents() would fail if service wasn't immediately connected 3. No permission check in startTask() when switching modes 4. Missing deviceControllerInstance getter in ComponentManager **Changes:** 1. Modified reinitializeAgent() to properly handle both SHIZUKU and ACCESSIBILITY modes 2. Removed early return in initializeAccessibilityComponents() - now initializes components regardless of immediate service connection 3. Added permission check in startTask() that verifies device controller is ready before execution 4. Added deviceControllerInstance property to ComponentManager for permission checking 5. Modified onServiceConnected() to only initialize Shizuku components when in SHIZUKU mode 6. Added informative toast messages when permissions are not granted **Testing:** Users can now: 1. Switch from Shizuku to Accessibility mode in settings 2. Start tasks once Accessibility Service is enabled 3. Receive clear error messages if permissions are not granted # Conflicts: # app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt # app/src/main/java/com/kevinluo/autoglm/MainActivity.kt --- app/src/main/AndroidManifest.xml | 14 + .../com/kevinluo/autoglm/ComponentManager.kt | 227 +++++++---- .../java/com/kevinluo/autoglm/MainActivity.kt | 365 ++++++++++++++---- .../AutoGLMAccessibilityService.kt | 337 ++++++++++++++++ .../device/AccessibilityDeviceController.kt | 271 +++++++++++++ .../autoglm/device/IDeviceController.kt | 171 ++++++++ .../autoglm/device/ShizukuDeviceController.kt | 101 +++++ .../autoglm/history/HistoryActivity.kt | 105 ++--- .../autoglm/history/HistoryDetailActivity.kt | 16 +- .../autoglm/screenshot/ScreenshotService.kt | 55 ++- .../autoglm/settings/SettingsActivity.kt | 50 ++- .../autoglm/settings/SettingsManager.kt | 57 ++- .../autoglm/ui/FloatingWindowService.kt | 4 +- app/src/main/res/layout/activity_main.xml | 94 ++++- app/src/main/res/layout/activity_settings.xml | 105 +++++ app/src/main/res/values/strings.xml | 53 ++- .../res/xml/accessibility_service_config.xml | 25 ++ 17 files changed, 1789 insertions(+), 261 deletions(-) create mode 100644 app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt create mode 100644 app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt create mode 100644 app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt create mode 100644 app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt create mode 100644 app/src/main/res/xml/accessibility_service_config.xml diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b4cd24b..e9b7593 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -118,6 +118,20 @@ android:name="android.view.im" android:resource="@xml/method_autoglm" /> + + + + + + + + \ No newline at end of file diff --git a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt index 3f29cf0..2ddcef7 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt @@ -7,7 +7,11 @@ import com.kevinluo.autoglm.agent.AgentConfig import com.kevinluo.autoglm.agent.PhoneAgent import com.kevinluo.autoglm.agent.PhoneAgentListener import com.kevinluo.autoglm.app.AppResolver +import com.kevinluo.autoglm.device.AccessibilityDeviceController +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.device.DeviceExecutor +import com.kevinluo.autoglm.device.IDeviceController +import com.kevinluo.autoglm.device.ShizukuDeviceController import com.kevinluo.autoglm.history.HistoryManager import com.kevinluo.autoglm.input.TextInputManager import com.kevinluo.autoglm.model.ModelClient @@ -22,24 +26,25 @@ import com.kevinluo.autoglm.util.Logger /** * Centralized component manager for dependency injection and lifecycle management. * Provides a single point of access for all major components in the application. - * + * * This class ensures: * - Proper dependency injection * - Lifecycle-aware component management * - Clean separation of concerns - * + * + * Requirements: All (integration) */ class ComponentManager private constructor(private val context: Context) { - + companion object { private const val TAG = "ComponentManager" - + @Volatile private var instance: ComponentManager? = null - + /** * Gets the singleton instance of ComponentManager. - * + * * @param context Application context * @return ComponentManager instance */ @@ -48,7 +53,7 @@ class ComponentManager private constructor(private val context: Context) { instance ?: ComponentManager(context.applicationContext).also { instance = it } } } - + /** * Clears the singleton instance. * Should be called when the application is being destroyed. @@ -60,66 +65,76 @@ class ComponentManager private constructor(private val context: Context) { } } } - + // Settings manager - always available val settingsManager: SettingsManager by lazy { SettingsManager(context) } - + // History manager - always available val historyManager: HistoryManager by lazy { HistoryManager.getInstance(context) } - + // User service reference - set when Shizuku connects private var userService: IUserService? = null - + + // Device controller - initialized based on control mode + private var _deviceController: IDeviceController? = null + // Lazily initialized components that depend on UserService private var _deviceExecutor: DeviceExecutor? = null private var _textInputManager: TextInputManager? = null private var _screenshotService: ScreenshotService? = null private var _actionHandler: ActionHandler? = null private var _phoneAgent: PhoneAgent? = null - + // Components that don't depend on UserService private var _modelClient: ModelClient? = null private var _appResolver: AppResolver? = null private var _swipeGenerator: HumanizedSwipeGenerator? = null - + /** * Checks if the UserService is connected. */ val isServiceConnected: Boolean get() = userService != null - + /** * Gets the DeviceExecutor instance. * Requires UserService to be connected. */ val deviceExecutor: DeviceExecutor? get() = _deviceExecutor - + + /** + * Gets the DeviceController instance. + * Available in both Shizuku and Accessibility modes. + */ + val deviceControllerInstance: IDeviceController? + get() = _deviceController + /** * Gets the ScreenshotService instance. * Requires UserService to be connected. */ val screenshotService: ScreenshotService? get() = _screenshotService - + /** * Gets the ActionHandler instance. * Requires UserService to be connected. */ val actionHandler: ActionHandler? get() = _actionHandler - + /** * Gets the PhoneAgent instance. * Requires UserService to be connected. */ val phoneAgent: PhoneAgent? get() = _phoneAgent - + /** * Gets the ModelClient instance. * Creates a new instance if config has changed. @@ -132,7 +147,7 @@ class ComponentManager private constructor(private val context: Context) { } return _modelClient!! } - + /** * Gets the AppResolver instance. */ @@ -143,7 +158,7 @@ class ComponentManager private constructor(private val context: Context) { } return _appResolver!! } - + /** * Gets the HumanizedSwipeGenerator instance. */ @@ -154,48 +169,53 @@ class ComponentManager private constructor(private val context: Context) { } return _swipeGenerator!! } - + // Track current model config for change detection private var currentModelConfig: ModelConfig? = null - + /** * Called when UserService connects. - * Initializes all service-dependent components. - * + * Initializes all service-dependent components for Shizuku mode. + * * @param service The connected UserService */ fun onServiceConnected(service: IUserService) { - Logger.i(TAG, "UserService connected, initializing components") + android.util.Log.i(TAG, "UserService connected, initializing components") userService = service - initializeServiceDependentComponents() + + // Only initialize Shizuku components if in Shizuku mode + val controlMode = settingsManager.getDeviceControlMode() + if (controlMode == DeviceControlMode.SHIZUKU) { + initializeServiceDependentComponents() + } } - + /** * Called when UserService disconnects. * Cleans up service-dependent components. */ fun onServiceDisconnected() { - Logger.i(TAG, "UserService disconnected, cleaning up components") + android.util.Log.i(TAG, "UserService disconnected, cleaning up components") userService = null cleanupServiceDependentComponents() } - + /** * Initializes components that depend on UserService. */ private fun initializeServiceDependentComponents() { val service = userService ?: return - + // Create DeviceExecutor _deviceExecutor = DeviceExecutor(service) - + // Create TextInputManager _textInputManager = TextInputManager(service) - + // Create ScreenshotService with floating window controller provider // Use a provider function so it can get the current instance dynamically _screenshotService = ScreenshotService(service) { FloatingWindowService.getInstance() } - + // Create ActionHandler with floating window provider to hide window during touch operations _actionHandler = ActionHandler( deviceExecutor = _deviceExecutor!!, @@ -204,7 +224,7 @@ class ComponentManager private constructor(private val context: Context) { textInputManager = _textInputManager!!, floatingWindowProvider = { FloatingWindowService.getInstance() } ) - + // Create PhoneAgent val agentConfig = settingsManager.getAgentConfig() _phoneAgent = PhoneAgent( @@ -214,10 +234,63 @@ class ComponentManager private constructor(private val context: Context) { config = agentConfig, historyManager = historyManager ) - - Logger.i(TAG, "All service-dependent components initialized") + + android.util.Log.i(TAG, "All service-dependent components initialized") } - + + /** + * Initializes components for accessibility mode. + * Creates a stub device controller when Accessibility Service is available. + */ + private fun initializeAccessibilityComponents() { + // Create AccessibilityDeviceController + _deviceController = AccessibilityDeviceController(context) + + // Create a stub UserService for accessibility mode + val stubUserService = object : IUserService { + override fun executeCommand(cmd: String): String = "" + override fun destroy() {} + override fun asBinder() = null + } + + // Create DeviceExecutor with stub service (won't be actually used) + _deviceExecutor = DeviceExecutor(stubUserService) + + // Create TextInputManager with stub service (won't be actually used) + _textInputManager = TextInputManager(stubUserService) + + // Create ScreenshotService for accessibility mode + _screenshotService = ScreenshotService(stubUserService) { FloatingWindowService.getInstance() } + + // Create ActionHandler for accessibility mode + // The DeviceExecutor and TextInputManager instances exist but their actual methods + // are not used in accessibility mode - all operations go through IDeviceController + _actionHandler = ActionHandler( + deviceExecutor = _deviceExecutor!!, + appResolver = appResolver, + swipeGenerator = swipeGenerator, + textInputManager = _textInputManager!!, + floatingWindowProvider = { FloatingWindowService.getInstance() } + ) + + // Create PhoneAgent + val agentConfig = settingsManager.getAgentConfig() + _phoneAgent = PhoneAgent( + modelClient = modelClient, + actionHandler = _actionHandler!!, + screenshotService = _screenshotService!!, + config = agentConfig, + historyManager = historyManager + ) + + // Check permission status and log accordingly + if (!_deviceController!!.checkPermission()) { + android.util.Log.w(TAG, "Accessibility service not yet connected, but components initialized for deferred startup") + } else { + android.util.Log.i(TAG, "Accessibility mode components initialized with service connected") + } + } + /** * Cleans up components that depend on UserService. */ @@ -228,68 +301,66 @@ class ComponentManager private constructor(private val context: Context) { _screenshotService = null _textInputManager = null _deviceExecutor = null - - Logger.i(TAG, "Service-dependent components cleaned up") + + android.util.Log.i(TAG, "Service-dependent components cleaned up") } - + /** * Reinitializes the PhoneAgent with updated configuration. * Call this after settings have been changed. - * - * Note: This will NOT reinitialize if a task is currently running or paused, - * to prevent accidentally cancelling user tasks. + * Handles mode switching between Shizuku and Accessibility modes. */ fun reinitializeAgent() { - if (userService == null) { - Logger.w(TAG, "Cannot reinitialize agent: UserService not connected") - return - } - - // Safety check: don't reinitialize while a task is active - _phoneAgent?.let { agent -> - if (agent.isRunning() || agent.isPaused()) { - Logger.w(TAG, "Cannot reinitialize agent: task is currently active (state: ${agent.getState()})") - return - } - } - - // Cancel any running task (should be IDLE at this point, but just in case) + val controlMode = settingsManager.getDeviceControlMode() + + // Cancel any running task _phoneAgent?.cancel() - + // Recreate model client with new config _modelClient = null - - // Recreate PhoneAgent - val agentConfig = settingsManager.getAgentConfig() - _phoneAgent = PhoneAgent( - modelClient = modelClient, - actionHandler = _actionHandler!!, - screenshotService = _screenshotService!!, - config = agentConfig, - historyManager = historyManager - ) - - Logger.i(TAG, "PhoneAgent reinitialized with new configuration") + + when (controlMode) { + DeviceControlMode.SHIZUKU -> { + if (userService == null) { + android.util.Log.w(TAG, "Cannot reinitialize agent: UserService not connected") + return + } + + // Re-initialize Shizuku components + cleanupServiceDependentComponents() + initializeServiceDependentComponents() + + android.util.Log.i(TAG, "PhoneAgent reinitialized for Shizuku mode") + } + + DeviceControlMode.ACCESSIBILITY -> { + // Re-initialize accessibility components + cleanupServiceDependentComponents() + initializeAccessibilityComponents() + + android.util.Log.i(TAG, "PhoneAgent reinitialized for Accessibility mode") + } + } } - + /** * Sets the listener for PhoneAgent events. - * + * * @param listener The listener to set */ fun setPhoneAgentListener(listener: PhoneAgentListener?) { _phoneAgent?.setListener(listener) } - + /** * Sets the confirmation callback for ActionHandler. - * + * * @param callback The callback to set */ fun setConfirmationCallback(callback: ActionHandler.ConfirmationCallback?) { _actionHandler?.setConfirmationCallback(callback) } - + /** * Checks if the model config has changed. */ @@ -300,20 +371,20 @@ class ComponentManager private constructor(private val context: Context) { } return changed } - + /** * Cleans up all components. * Should be called when the application is being destroyed. */ fun cleanup() { - Logger.i(TAG, "Cleaning up all components") + android.util.Log.i(TAG, "Cleaning up all components") cleanupServiceDependentComponents() _modelClient = null _appResolver = null _swipeGenerator = null currentModelConfig = null } - + /** * Gets the current state summary for debugging. */ diff --git a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt index c1fe57d..5ba9594 100644 --- a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt @@ -1,13 +1,17 @@ package com.kevinluo.autoglm +import android.accessibilityservice.AccessibilityServiceInfo import android.content.ComponentName import android.content.Intent import android.content.ServiceConnection import android.content.pm.PackageManager import android.graphics.drawable.GradientDrawable +import android.os.Build import android.os.Bundle import android.os.IBinder +import android.provider.Settings import android.view.View +import android.view.accessibility.AccessibilityManager import android.widget.Button import android.widget.ImageButton import android.widget.TextView @@ -19,10 +23,12 @@ import androidx.core.content.ContextCompat import androidx.core.view.ViewCompat import androidx.core.view.WindowInsetsCompat import androidx.lifecycle.lifecycleScope +import com.kevinluo.autoglm.accessibility.AutoGLMAccessibilityService import com.kevinluo.autoglm.action.ActionHandler import com.kevinluo.autoglm.action.AgentAction import com.kevinluo.autoglm.agent.PhoneAgent import com.kevinluo.autoglm.agent.PhoneAgentListener +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.settings.SettingsActivity import com.kevinluo.autoglm.ui.FloatingWindowService import com.kevinluo.autoglm.ui.TaskStatus @@ -50,10 +56,12 @@ import rikka.shizuku.Shizuku * The activity implements [PhoneAgentListener] to receive callbacks * during task execution for UI updates. * + * Requirements: 1.1, 2.1, 2.2 */ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Shizuku status views + private lateinit var shizukuCard: View private lateinit var statusText: TextView private lateinit var shizukuStatusIndicator: View private lateinit var shizukuButtonsRow: View @@ -73,6 +81,12 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private lateinit var keyboardStatusText: TextView private lateinit var enableKeyboardBtn: Button + // Accessibility permission views + private lateinit var accessibilityCard: View + private lateinit var accessibilityStatusIcon: android.widget.ImageView + private lateinit var accessibilityStatusText: TextView + private lateinit var requestAccessibilityBtn: Button + // Task input views private lateinit var taskInputLayout: TextInputLayout private lateinit var taskInput: TextInputEditText @@ -88,7 +102,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Component manager for dependency injection private lateinit var componentManager: ComponentManager - + // Current step tracking for floating window private var currentStepNumber = 0 private var currentThinking = "" @@ -108,26 +122,26 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { override fun onServiceConnected(name: ComponentName?, service: IBinder?) { val userService = IUserService.Stub.asInterface(service) Logger.i(TAG, "UserService connected") - + // Notify ComponentManager componentManager.onServiceConnected(userService) - + runOnUiThread { Toast.makeText(this@MainActivity, R.string.toast_user_service_connected, Toast.LENGTH_SHORT).show() - updateShizukuStatus() + updateDeviceServiceStatus() initializePhoneAgent() } } override fun onServiceDisconnected(name: ComponentName?) { Logger.i(TAG, "UserService disconnected") - + // Notify ComponentManager componentManager.onServiceDisconnected() - + runOnUiThread { Toast.makeText(this@MainActivity, R.string.toast_user_service_disconnected, Toast.LENGTH_SHORT).show() - updateShizukuStatus() + updateDeviceServiceStatus() updateTaskButtonStates() } } @@ -137,7 +151,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Shizuku.OnRequestPermissionResultListener { requestCode, grantResult -> if (requestCode == SHIZUKU_PERMISSION_REQUEST_CODE) { if (grantResult == PackageManager.PERMISSION_GRANTED) { - updateShizukuStatus() + updateDeviceServiceStatus() bindUserService() Toast.makeText(this, R.string.toast_shizuku_permission_granted, Toast.LENGTH_SHORT).show() } else { @@ -147,7 +161,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } private val binderReceivedListener = Shizuku.OnBinderReceivedListener { - updateShizukuStatus() + updateDeviceServiceStatus() if (hasShizukuPermission()) { bindUserService() } @@ -156,10 +170,19 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private val binderDeadListener = Shizuku.OnBinderDeadListener { Logger.w(TAG, "Shizuku binder died") componentManager.onServiceDisconnected() - updateShizukuStatus() + updateDeviceServiceStatus() updateTaskButtonStates() } + // Accessibility state change listener + private val accessibilityStateChangeListener = + AccessibilityManager.AccessibilityStateChangeListener { enabled -> + Logger.d(TAG, "Accessibility state changed: enabled=$enabled") + runOnUiThread { + updateAccessibilityStatus() + } + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() @@ -174,17 +197,54 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Initialize ComponentManager componentManager = ComponentManager.getInstance(this) Logger.i(TAG, "ComponentManager initialized") - + + // Log all launchable apps at startup + logAllLaunchableApps() + initViews() setupListeners() setupShizukuListeners() - - updateShizukuStatus() + setupAccessibilityListener() + + updateDeviceServiceStatus() updateOverlayPermissionStatus() updateKeyboardStatus() + updateAccessibilityStatus() updateTaskStatus(TaskStatus.IDLE) } - + + /** + * Logs all launchable apps for debugging purposes. + * + * Queries the package manager for all apps with launcher activities + * and logs them for debugging. Only logs the first 20 apps to avoid + * excessive log output. + */ + private fun logAllLaunchableApps() { + // Query apps without loading icons to avoid excessive logging + val intent = android.content.Intent(android.content.Intent.ACTION_MAIN).apply { + addCategory(android.content.Intent.CATEGORY_LAUNCHER) + } + val resolveInfoList = packageManager.queryIntentActivities(intent, 0) + + val apps = resolveInfoList.mapNotNull { resolveInfo -> + val activityInfo = resolveInfo.activityInfo ?: return@mapNotNull null + val displayName = resolveInfo.loadLabel(packageManager)?.toString() ?: return@mapNotNull null + val packageName = activityInfo.packageName ?: return@mapNotNull null + displayName to packageName + }.distinctBy { it.second } + + Logger.i(TAG, "=== All Launchable Apps: ${apps.size} total ===") + // Only log first 20 apps to avoid log quota + apps.take(20).forEach { (name, pkg) -> + Logger.i(TAG, " $name -> $pkg") + } + if (apps.size > 20) { + Logger.i(TAG, " ... and ${apps.size - 20} more apps") + } + Logger.i(TAG, "=== End of App List ===") + } + /** * Updates the overlay permission status display. * @@ -193,7 +253,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun updateOverlayPermissionStatus() { val hasPermission = FloatingWindowService.canDrawOverlays(this) - + if (hasPermission) { overlayStatusText.text = getString(R.string.overlay_permission_granted) overlayStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_running)) @@ -204,7 +264,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { requestOverlayBtn.visibility = View.VISIBLE } } - + /** * Updates the keyboard status display. * @@ -212,7 +272,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun updateKeyboardStatus() { val status = com.kevinluo.autoglm.input.KeyboardHelper.getAutoGLMKeyboardStatus(this) - + when (status) { com.kevinluo.autoglm.input.KeyboardHelper.KeyboardStatus.ENABLED -> { keyboardStatusText.text = getString(R.string.keyboard_settings_subtitle) @@ -228,16 +288,68 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } } + /** + * Updates the accessibility service status display. + * + * Checks if AutoGLM Accessibility Service is enabled and updates the UI accordingly. + * Also shows/hides relevant permission cards based on the device control mode. + */ + private fun updateAccessibilityStatus() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + + // Show/hide relevant cards based on control mode + when (controlMode) { + DeviceControlMode.SHIZUKU -> { + // Shizuku mode: show Shizuku card and keyboard card, hide accessibility card + shizukuCard.visibility = View.VISIBLE + keyboardCard.visibility = View.VISIBLE + accessibilityCard.visibility = View.GONE + } + DeviceControlMode.ACCESSIBILITY -> { + // Accessibility mode: hide Shizuku card and keyboard card, show accessibility card + shizukuCard.visibility = View.GONE + keyboardCard.visibility = View.GONE + accessibilityCard.visibility = View.VISIBLE + + // Update accessibility status + val isAccessibilityEnabled = AutoGLMAccessibilityService.isEnabled() + + if (isAccessibilityEnabled) { + accessibilityStatusText.text = getString(R.string.accessibility_permission_granted) + accessibilityStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_running)) + requestAccessibilityBtn.visibility = View.GONE + } else { + accessibilityStatusText.text = getString(R.string.accessibility_permission_denied) + accessibilityStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_waiting)) + requestAccessibilityBtn.visibility = View.VISIBLE + requestAccessibilityBtn.text = getString(R.string.request_accessibility_permission) + } + + // Check Android version for accessibility support + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + accessibilityStatusText.text = getString(R.string.accessibility_not_supported) + requestAccessibilityBtn.visibility = View.GONE + } + } + } + + // Also update the main status display to reflect the current mode + updateDeviceServiceStatus() + } + override fun onResume() { super.onResume() Logger.d(TAG, "onResume - checking for settings changes") - + // Update overlay permission status (user may have granted it) updateOverlayPermissionStatus() - + // Update keyboard status (user may have enabled it) updateKeyboardStatus() - + + // Update accessibility status (user may have enabled it or mode changed) + updateAccessibilityStatus() + // Re-setup floating window callbacks if service is running FloatingWindowService.getInstance()?.let { service -> service.setStopTaskCallback { @@ -260,16 +372,11 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { resumeTask() } } - + // Only reinitialize if service is connected and we need to refresh if (componentManager.isServiceConnected) { // Check if settings actually changed before reinitializing - // But NEVER reinitialize while a task is running or paused - this would cancel the task! - val isTaskActive = componentManager.phoneAgent?.let { - it.isRunning() || it.isPaused() - } ?: false - - if (!isTaskActive && componentManager.settingsManager.hasConfigChanged()) { + if (componentManager.settingsManager.hasConfigChanged()) { componentManager.reinitializeAgent() } componentManager.setPhoneAgentListener(this) @@ -281,15 +388,20 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { override fun onDestroy() { Logger.i(TAG, "onDestroy - cleaning up") super.onDestroy() - + // Remove Shizuku listeners Shizuku.removeRequestPermissionResultListener(onRequestPermissionResultListener) Shizuku.removeBinderReceivedListener(binderReceivedListener) Shizuku.removeBinderDeadListener(binderDeadListener) + // Remove accessibility state change listener + (getSystemService(ACCESSIBILITY_SERVICE) as? AccessibilityManager)?.removeAccessibilityStateChangeListener( + accessibilityStateChangeListener + ) + // Cancel any running task componentManager.phoneAgent?.cancel() - + // Unbind user service if (componentManager.isServiceConnected) { try { @@ -298,7 +410,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Logger.e(TAG, "Error unbinding user service", e) } } - + // Note: Don't stop FloatingWindowService here - it should run independently // The service will be stopped when user explicitly closes it or the app process is killed } @@ -310,6 +422,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun initViews() { // Shizuku status views + shizukuCard = findViewById(R.id.shizukuCard) statusText = findViewById(R.id.statusText) shizukuStatusIndicator = findViewById(R.id.shizukuStatusIndicator) shizukuButtonsRow = findViewById(R.id.shizukuButtonsRow) @@ -329,6 +442,12 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { keyboardStatusText = findViewById(R.id.keyboardStatusText) enableKeyboardBtn = findViewById(R.id.enableKeyboardBtn) + // Accessibility permission views + accessibilityCard = findViewById(R.id.accessibilityCard) + accessibilityStatusIcon = findViewById(R.id.accessibilityStatusIcon) + accessibilityStatusText = findViewById(R.id.accessibilityStatusText) + requestAccessibilityBtn = findViewById(R.id.requestAccessibilityBtn) + // Task input views taskInputLayout = findViewById(R.id.taskInputLayout) taskInput = findViewById(R.id.taskInput) @@ -359,7 +478,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { openShizukuApp() } - // Settings button + // Settings button - Requirements: 6.1 settingsBtn.setOnClickListener { startActivity(Intent(this, SettingsActivity::class.java)) } @@ -385,16 +504,23 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { com.kevinluo.autoglm.input.KeyboardHelper.openInputMethodSettings(this) } - // Start task button + // Accessibility permission button + requestAccessibilityBtn.setOnClickListener { + // Open accessibility settings + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + startActivity(intent) + } + + // Start task button - Requirements: 1.1 startTaskBtn.setOnClickListener { startTask() } - // Cancel task button + // Cancel task button - Requirements: 1.4 cancelTaskBtn.setOnClickListener { cancelTask() } - + // Select template button btnSelectTemplate.setOnClickListener { showTemplateSelectionDialog() @@ -404,7 +530,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { taskInput.setOnFocusChangeListener { _, _ -> updateTaskButtonStates() } - + // Watch for text changes to enable/disable start button taskInput.addTextChangedListener(object : android.text.TextWatcher { override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {} @@ -541,7 +667,17 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Shizuku.addBinderReceivedListenerSticky(binderReceivedListener) Shizuku.addBinderDeadListener(binderDeadListener) } - + + /** + * Sets up accessibility state change listener. + * + * Registers listener to detect when accessibility service is enabled/disabled. + */ + private fun setupAccessibilityListener() { + val accessibilityManager = getSystemService(ACCESSIBILITY_SERVICE) as? AccessibilityManager + accessibilityManager?.addAccessibilityStateChangeListener(accessibilityStateChangeListener) + } + /** * Opens the Shizuku app or Play Store if not installed. * @@ -573,23 +709,24 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Called after UserService is connected. Sets up the agent listener * and confirmation callback for sensitive operations. * + * Requirements: 1.1, 2.1, 2.2 */ private fun initializePhoneAgent() { if (!componentManager.isServiceConnected) { Logger.w(TAG, "Cannot initialize PhoneAgent: service not connected") return } - + // Set up listener componentManager.setPhoneAgentListener(this) - + // Setup confirmation callback setupConfirmationCallback() - + updateTaskButtonStates() Logger.i(TAG, "PhoneAgent initialized successfully") } - + /** * Sets up the confirmation callback for sensitive operations. * @@ -635,31 +772,50 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Validates the task description, checks agent state, starts the * floating window service, and launches the task in a coroutine. * + * Requirements: 1.1, 2.1, 2.2 */ private fun startTask() { val taskDescription = taskInput.text?.toString()?.trim() ?: "" - + // Validate task description if (taskDescription.isBlank()) { Toast.makeText(this, R.string.toast_task_empty, Toast.LENGTH_SHORT).show() taskInputLayout.error = getString(R.string.toast_task_empty) return } - + taskInputLayout.error = null - + val agent = componentManager.phoneAgent if (agent == null) { Logger.e(TAG, "PhoneAgent not initialized") + Toast.makeText(this, "Device controller not initialized. Please check your settings.", Toast.LENGTH_SHORT).show() return } - + // Check if already running if (agent.isRunning()) { Logger.w(TAG, "Task already running") return } - + + // Check device controller permissions + val deviceController = componentManager.deviceControllerInstance + if (deviceController != null && !deviceController.checkPermission()) { + val mode = deviceController.getMode() + val message = when (mode) { + com.kevinluo.autoglm.device.DeviceControlMode.ACCESSIBILITY -> { + "Accessibility service not enabled. Please enable it in system settings." + } + com.kevinluo.autoglm.device.DeviceControlMode.SHIZUKU -> { + "Shizuku service not connected. Please ensure Shizuku is running." + } + } + Toast.makeText(this, message, Toast.LENGTH_LONG).show() + deviceController.requestPermission(this) + return + } + // Start floating window service if overlay permission granted if (FloatingWindowService.canDrawOverlays(this)) { Logger.d(TAG, "startTask: Starting floating window service") @@ -669,7 +825,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { FloatingWindowService.requestOverlayPermission(this) return } - + // Update UI state - manually set running state since agent.run() hasn't started yet updateTaskStatus(TaskStatus.RUNNING) // Manually update UI for running state @@ -677,9 +833,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { runningSection.visibility = View.VISIBLE cancelTaskBtn.isEnabled = true taskInput.isEnabled = false - + Logger.i(TAG, "Starting task: $taskDescription") - + // Run task in coroutine lifecycleScope.launch { // Set up callbacks immediately after service starts @@ -711,15 +867,15 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { floatingWindow?.updateStatus(TaskStatus.RUNNING) floatingWindow?.show() } - + // Minimize app to let agent work withContext(Dispatchers.Main) { moveTaskToBack(true) } - + try { val result = agent.run(taskDescription) - + withContext(Dispatchers.Main) { if (result.success) { Logger.i(TAG, "Task completed: ${result.message}") @@ -746,26 +902,27 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Cancels the agent, resets its state, and updates the UI * to reflect the cancelled status. * + * Requirements: 1.1, 2.1, 2.2 */ private fun cancelTask() { Logger.i(TAG, "Cancelling task") - + // Cancel the agent - this will cancel any ongoing network requests componentManager.phoneAgent?.cancel() - + // Reset the agent state so it can accept new tasks componentManager.phoneAgent?.reset() - + Toast.makeText(this, R.string.toast_task_cancelled, Toast.LENGTH_SHORT).show() updateTaskStatus(TaskStatus.FAILED) updateTaskButtonStates() - + // Update floating window to show cancelled state // Use the same message as PhoneAgent for consistency val cancellationMessage = PhoneAgent.CANCELLATION_MESSAGE FloatingWindowService.getInstance()?.showResult(cancellationMessage, false) } - + /** * Pauses the currently running task. * @@ -773,14 +930,14 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun pauseTask() { Logger.i(TAG, "Pausing task") - + val paused = componentManager.phoneAgent?.pause() == true if (paused) { updateTaskStatus(TaskStatus.PAUSED) FloatingWindowService.getInstance()?.updateStatus(TaskStatus.PAUSED) } } - + /** * Resumes a paused task. * @@ -788,7 +945,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun resumeTask() { Logger.i(TAG, "Resuming task") - + val resumed = componentManager.phoneAgent?.resume() == true if (resumed) { updateTaskStatus(TaskStatus.RUNNING) @@ -807,15 +964,15 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { val hasAgent = componentManager.phoneAgent != null val hasTaskText = !taskInput.text.isNullOrBlank() val isRunning = componentManager.phoneAgent?.isRunning() == true - + // Show/hide sections based on running state startTaskBtn.visibility = if (isRunning) View.GONE else View.VISIBLE runningSection.visibility = if (isRunning) View.VISIBLE else View.GONE - + startTaskBtn.isEnabled = hasService && hasAgent && hasTaskText && !isRunning cancelTaskBtn.isEnabled = isRunning taskInput.isEnabled = !isRunning - + Logger.d( TAG, "Button states updated: service=$hasService, agent=$hasAgent, " + @@ -831,6 +988,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param status The new task status to display * + * Requirements: 1.1, 2.1, 2.2 */ private fun updateTaskStatus(status: TaskStatus) { val (text, colorRes) = when (status) { @@ -842,14 +1000,14 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { TaskStatus.WAITING_CONFIRMATION -> "等待确认" to R.color.status_waiting TaskStatus.WAITING_TAKEOVER -> "等待接管" to R.color.status_waiting } - + taskStatusText.text = text - + val drawable = taskStatusIndicator.background as? GradientDrawable ?: GradientDrawable().also { taskStatusIndicator.background = it } drawable.shape = GradientDrawable.OVAL drawable.setColor(ContextCompat.getColor(this, colorRes)) - + // Also update floating window FloatingWindowService.getInstance()?.updateStatus(status) } @@ -863,6 +1021,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param stepNumber The current step number * + * Requirements: 1.1, 2.1, 2.2 */ override fun onStepStarted(stepNumber: Int) { runOnUiThread { @@ -880,6 +1039,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param thinking The model's thinking text * + * Requirements: 1.1, 2.1, 2.2 */ override fun onThinkingUpdate(thinking: String) { runOnUiThread { @@ -894,6 +1054,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param action The action that was executed * + * Requirements: 1.1, 2.1, 2.2 */ override fun onActionExecuted(action: AgentAction) { runOnUiThread { @@ -910,6 +1071,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param message The completion message * + * Requirements: 1.1, 2.1, 2.2 */ override fun onTaskCompleted(message: String) { runOnUiThread { @@ -928,6 +1090,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param error The error message * + * Requirements: 1.1, 2.1, 2.2 */ override fun onTaskFailed(error: String) { runOnUiThread { @@ -943,6 +1106,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * Note: Floating window hide is handled by ScreenshotService. * + * Requirements: 1.1, 2.1, 2.2 */ override fun onScreenshotStarted() { // Floating window hide is handled by ScreenshotService @@ -953,6 +1117,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * Note: Floating window show is handled by ScreenshotService. * + * Requirements: 1.1, 2.1, 2.2 */ override fun onScreenshotCompleted() { // Floating window show is handled by ScreenshotService @@ -972,7 +1137,22 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // endregion - // region Shizuku Methods + // region Device Service Status Methods + + /** + * Updates the device service status display based on current control mode. + * + * For Shizuku mode: shows Shizuku connection status. + * For Accessibility mode: shows Accessibility service status. + */ + private fun updateDeviceServiceStatus() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + + when (controlMode) { + DeviceControlMode.SHIZUKU -> updateShizukuStatusDisplay() + DeviceControlMode.ACCESSIBILITY -> updateAccessibilityStatusDisplay() + } + } /** * Updates the Shizuku connection status display. @@ -980,7 +1160,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Checks Shizuku binder status, permission, and service connection, * then updates the UI to reflect the current state. */ - private fun updateShizukuStatus() { + private fun updateShizukuStatusDisplay() { val isBinderAlive = try { Shizuku.pingBinder() } catch (e: Exception) { @@ -996,7 +1176,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { !serviceConnected -> getString(R.string.shizuku_status_connecting) else -> getString(R.string.shizuku_status_connected) } - + val statusColor = when { !isBinderAlive -> R.color.status_failed !hasPermission -> R.color.status_waiting @@ -1007,7 +1187,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { runOnUiThread { statusText.text = statusMessage shizukuStatusIndicator.background.setTint(getColor(statusColor)) - + // Show buttons based on Shizuku state if (serviceConnected) { // Connected - hide buttons row @@ -1021,7 +1201,40 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { requestPermissionBtn.visibility = View.VISIBLE requestPermissionBtn.isEnabled = isBinderAlive } - + + updateTaskButtonStates() + } + } + + /** + * Updates the Accessibility service status display. + * + * Shows the accessibility service connection status in the same location + * where Shizuku status is normally displayed. + */ + private fun updateAccessibilityStatusDisplay() { + val isAccessibilityEnabled = AutoGLMAccessibilityService.isEnabled() + val isAndroidSupported = Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU + + val statusMessage = when { + !isAndroidSupported -> getString(R.string.accessibility_not_supported) + !isAccessibilityEnabled -> getString(R.string.accessibility_permission_denied) + else -> getString(R.string.accessibility_permission_granted) + } + + val statusColor = when { + !isAndroidSupported -> R.color.status_failed + !isAccessibilityEnabled -> R.color.status_waiting + else -> R.color.status_running + } + + runOnUiThread { + statusText.text = statusMessage + shizukuStatusIndicator.background.setTint(getColor(statusColor)) + + // Hide Shizuku buttons in accessibility mode + shizukuButtonsRow.visibility = View.GONE + updateTaskButtonStates() } } @@ -1098,9 +1311,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private fun formatAction(action: AgentAction): String = action.formatForDisplay() // endregion - + // region Task Templates - + /** * Shows a dialog to select a task template. * @@ -1109,7 +1322,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun showTemplateSelectionDialog() { val templates = componentManager.settingsManager.getTaskTemplates() - + if (templates.isEmpty()) { Toast.makeText(this, R.string.settings_no_templates, Toast.LENGTH_SHORT).show() // Offer to go to settings to add templates @@ -1123,9 +1336,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { .show() return } - + val templateNames = templates.map { it.name }.toTypedArray() - + AlertDialog.Builder(this) .setTitle(R.string.task_select_template) .setItems(templateNames) { _, which -> @@ -1136,7 +1349,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + // endregion companion object { diff --git a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt new file mode 100644 index 0000000..04b9b9b --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt @@ -0,0 +1,337 @@ +package com.kevinluo.autoglm.accessibility + +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.GestureDescription +import android.content.Intent +import android.graphics.Bitmap +import android.graphics.Path +import android.os.Build +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo + +import com.kevinluo.autoglm.util.Logger +import kotlinx.coroutines.suspendCancellableCoroutine +import java.util.concurrent.atomic.AtomicReference +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +/** + * Accessibility Service for device control without Shizuku. + * + * This service provides: + * - Screen capture using takeScreenshot() API (Android 13+) + * - Touch operations via dispatchGesture() + * - Text input via AccessibilityNodeInfo + * - Key press simulation + * - App launching and info retrieval + * + * Requirements: Android 13+ (API 33) for screenshot functionality + */ +class AutoGLMAccessibilityService : AccessibilityService() { + + companion object { + private const val TAG = "AutoGLMAccessibilityService" + + // Gesture timing constants + private const val TAP_DURATION_MS = 100L + private const val DOUBLE_TAP_INTERVAL_MS = 100L + + // Android KeyEvent keycodes + const val KEYCODE_BACK = 4 + const val KEYCODE_HOME = 3 + const val KEYCODE_RECENTS = 187 + const val KEYCODE_NOTIFICATIONS = 4 + const val KEYCODE_QUICK_SETTINGS = 5 + + @Volatile + private var instance: AutoGLMAccessibilityService? = null + + /** + * Gets the singleton instance of the accessibility service. + * + * @return The service instance, or null if not connected + */ + fun getInstance(): AutoGLMAccessibilityService? = instance + + /** + * Checks if the accessibility service is enabled. + * + * @return true if the service is running, false otherwise + */ + fun isEnabled(): Boolean = instance != null + } + + private val serviceState = AtomicReference(ServiceState.IDLE) + + private enum class ServiceState { + IDLE, CONNECTING, CONNECTED, DISCONNECTED + } + + override fun onServiceConnected() { + super.onServiceConnected() + Logger.i(TAG, "AccessibilityService connected") + instance = this + serviceState.set(ServiceState.CONNECTED) + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + // Handle accessibility events if needed + // Currently not used, but can be used for detecting UI changes + } + + override fun onInterrupt() { + Logger.w(TAG, "AccessibilityService interrupted") + } + + override fun onDestroy() { + super.onDestroy() + Logger.i(TAG, "AccessibilityService destroyed") + instance = null + serviceState.set(ServiceState.DISCONNECTED) + } + + // ==================== Screenshot ==================== + + /** + * Captures the current screen content. + * + * Uses AccessibilityService.takeScreenshot() API available on Android 13+. + * + * @return Bitmap of the screen, or null if capture failed + */ + suspend fun captureScreen(): Bitmap? = suspendCancellableCoroutine { continuation -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + try { + takeScreenshot(/* displayId= */ 0, /* executor= */ mainExecutor, + object : android.accessibilityservice.AccessibilityService.TakeScreenshotCallback { + override fun onSuccess(screenshot: android.accessibilityservice.AccessibilityService.ScreenshotResult) { + try { + val hardwareBuffer = screenshot.hardwareBuffer + val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, screenshot.colorSpace) + if (bitmap != null) { + continuation.resume(bitmap) + } else { + continuation.resume(null) + } + hardwareBuffer.close() + } catch (e: Exception) { + Logger.e(TAG, "Failed to process screenshot result", e) + continuation.resume(null) + } + } + + override fun onFailure(error: Int) { + Logger.e(TAG, "Screenshot capture failed with error code: $error") + continuation.resume(null) + } + }) + } catch (e: Exception) { + Logger.e(TAG, "Failed to capture screenshot", e) + continuation.resume(null) + } + } else { + Logger.w(TAG, "Screenshot not supported on Android < 13") + continuation.resume(null) + } + } + + // ==================== Touch Operations ==================== + + /** + * Performs a tap gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @return true if gesture was dispatched successfully + */ + suspend fun tap(x: Int, y: Int): Boolean { + val path = Path().apply { + moveTo(x.toFloat(), y.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, TAP_DURATION_MS)) + .build() + + return dispatchGesture(gesture, null, null) + } + + /** + * Performs a double tap gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @return true if both taps were dispatched successfully + */ + suspend fun doubleTap(x: Int, y: Int): Boolean { + val firstTap = tap(x, y) + kotlinx.coroutines.delay(DOUBLE_TAP_INTERVAL_MS) + val secondTap = tap(x, y) + return firstTap && secondTap + } + + /** + * Performs a long press gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @param durationMs Duration of the long press in milliseconds + * @return true if gesture was dispatched successfully + */ + suspend fun longPress(x: Int, y: Int, durationMs: Int): Boolean { + val path = Path().apply { + moveTo(x.toFloat(), y.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs.toLong())) + .build() + + return dispatchGesture(gesture, null, null) + } + + /** + * Performs a swipe gesture from start to end coordinates. + * + * @param startX Start X coordinate in pixels + * @param startY Start Y coordinate in pixels + * @param endX End X coordinate in pixels + * @param endY End Y coordinate in pixels + * @param durationMs Duration of the swipe in milliseconds + * @return true if gesture was dispatched successfully + */ + suspend fun swipe(startX: Int, startY: Int, endX: Int, endY: Int, durationMs: Int): Boolean { + val path = Path().apply { + moveTo(startX.toFloat(), startY.toFloat()) + lineTo(endX.toFloat(), endY.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs.toLong())) + .build() + + return dispatchGesture(gesture, null, null) + } + + // ==================== Key Press ==================== + + /** + * Performs a key press event. + * + * Uses performGlobalAction for system keys and fallback to dispatchKeyEvent. + * + * @param keyCode Android KeyEvent keycode + * @return true if key press was successful + */ + fun pressKey(keyCode: Int): Boolean { + return when (keyCode) { + KEYCODE_BACK -> performGlobalAction(GLOBAL_ACTION_BACK) + KEYCODE_HOME -> performGlobalAction(GLOBAL_ACTION_HOME) + KEYCODE_RECENTS -> performGlobalAction(GLOBAL_ACTION_RECENTS) + KEYCODE_NOTIFICATIONS -> performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS) + KEYCODE_QUICK_SETTINGS -> performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS) + else -> { + Logger.w(TAG, "Key code $keyCode not supported via accessibility") + false + } + } + } + + // ==================== Text Input ==================== + + /** + * Inputs text into the currently focused editable field. + * + * @param text The text to input + * @return true if text was entered successfully + */ + fun inputText(text: String): Boolean { + val rootNode = rootInActiveWindow ?: run { + Logger.w(TAG, "No active window root node") + return false + } + + val focusedNode = findFocusedEditableNode(rootNode) ?: run { + Logger.w(TAG, "No focused editable node found") + return false + } + + return try { + // Clear existing text + focusedNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, createSetTextBundle("")) + + // Set new text + focusedNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, createSetTextBundle(text)) + + Logger.d(TAG, "Text input successful: '${text.take(30)}...'") + true + } catch (e: Exception) { + Logger.e(TAG, "Failed to input text", e) + false + } + } + + /** + * Finds the currently focused editable node in the node tree. + * + * @param node The root node to search from + * @return The focused editable node, or null if not found + */ + private fun findFocusedEditableNode(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { + // Check if this node is focused and editable + if (node.isFocused && node.isEditable) { + return node + } + + // Recursively search children + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + val result = findFocusedEditableNode(child) + if (result != null) { + return result + } + // Note: AccessibilityNodeInfo.recycle() is deprecated and no longer needed + // The garbage collector will handle cleanup automatically + } + + return null + } + + /** + * Creates a bundle for setting text in an editable node. + * + * @param text The text to set + * @return Bundle with text argument + */ + private fun createSetTextBundle(text: String): android.os.Bundle { + return android.os.Bundle().apply { + putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + } + } + + // ==================== App Operations ==================== + + /** + * Gets the current foreground app's package name. + * + * @return Package name of the current app, or empty string if not found + */ + fun getCurrentApp(): String { + val rootNode = rootInActiveWindow ?: return "" + + // Try to get package name from window info + val packageName = rootNode.packageName?.toString() + + return packageName ?: "" + } + + /** + * Opens the accessibility settings for this app. + * + * @param intent The intent to start the accessibility settings + */ + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + // Handle any commands sent to the service + return START_STICKY + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt new file mode 100644 index 0000000..e882e05 --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt @@ -0,0 +1,271 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import android.content.Intent +import android.graphics.Bitmap +import android.os.Build +import android.provider.Settings +import com.kevinluo.autoglm.accessibility.AutoGLMAccessibilityService +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.util.Logger +import com.kevinluo.autoglm.util.Point +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import android.util.Base64 + +/** + * Device controller implementation using Android Accessibility Service. + * + * This controller uses the AccessibilityService API for device control. + * It requires the accessibility service to be enabled in system settings. + * + * @param context Android context + * + * Requirements: Android 13+ (API 33) for screenshot functionality + */ +class AccessibilityDeviceController( + private val context: Context +) : IDeviceController { + + override fun getMode(): DeviceControlMode = DeviceControlMode.ACCESSIBILITY + + override fun checkPermission(): Boolean { + return AutoGLMAccessibilityService.isEnabled() + } + + override fun requestPermission(context: Context): Boolean { + try { + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(intent) + return true + } catch (e: Exception) { + Logger.e(TAG, "Failed to open accessibility settings", e) + return false + } + } + + override suspend fun tap(x: Int, y: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.tap(x, y) + if (result) { + "Tap at ($x, $y) succeeded" + } else { + "Tap at ($x, $y) failed" + } + } + + override suspend fun doubleTap(x: Int, y: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.doubleTap(x, y) + if (result) { + "Double tap at ($x, $y) succeeded" + } else { + "Double tap at ($x, $y) failed" + } + } + + override suspend fun longPress(x: Int, y: Int, durationMs: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.longPress(x, y, durationMs) + if (result) { + "Long press at ($x, $y) for ${durationMs}ms succeeded" + } else { + "Long press at ($x, $y) failed" + } + } + + override suspend fun swipe(points: List, durationMs: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + if (points.size < 2) { + return@withContext "Error: Swipe requires at least 2 points" + } + + // Perform swipe between each consecutive point + var allSuccess = true + for (i in 0 until points.size - 1) { + val start = points[i] + val end = points[i + 1] + val segmentDuration = durationMs / (points.size - 1) + + val result = service.swipe(start.x, start.y, end.x, end.y, segmentDuration) + if (!result) { + allSuccess = false + } + + // Small delay between segments for smooth multi-point swipe + if (i < points.size - 2) { + kotlinx.coroutines.delay(20) + } + } + + if (allSuccess) { + "Swipe with ${points.size} points succeeded" + } else { + "Swipe completed with some segments failed" + } + } + + override suspend fun pressKey(keyCode: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.pressKey(keyCode) + if (result) { + "Key press $keyCode succeeded" + } else { + "Key press $keyCode failed" + } + } + + override suspend fun launchApp(packageName: String): String = withContext(Dispatchers.Main) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + // For accessibility mode, we can't directly launch apps via shell + // We need to use a different approach or return an error + // One option is to use PackageManager to get the launch intent + try { + val pm = context.packageManager + val intent = pm.getLaunchIntentForPackage(packageName) + if (intent != null) { + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(intent) + "Launched app: $packageName" + } else { + "Error: No launch intent found for $packageName" + } + } catch (e: Exception) { + Logger.e(TAG, "Failed to launch app: $packageName", e) + "Error: Failed to launch $packageName: ${e.message}" + } + } else { + "Error: App launch not supported on Android < 5.0" + } + } + + override suspend fun getCurrentApp(): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "" + } + + service.getCurrentApp() + } + + override suspend fun inputText(text: String): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.inputText(text) + if (result) { + "Text input successful: '${text.take(30)}...'" + } else { + "Text input failed: No focused editable field" + } + } + + override suspend fun captureScreen(): Screenshot = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + Logger.e(TAG, "Accessibility service not connected") + return@withContext createFallbackScreenshot() + } + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + Logger.w(TAG, "Screenshot not supported on Android < 13") + return@withContext createFallbackScreenshot() + } + + try { + val bitmap = service.captureScreen() + if (bitmap != null) { + val width = bitmap.width + val height = bitmap.height + + // Convert bitmap to Screenshot + val base64Data = encodeBitmapToBase64(bitmap) + bitmap.recycle() + + Screenshot( + base64Data = base64Data, + width = width, + height = height, + originalWidth = width, + originalHeight = height, + isSensitive = false + ) + } else { + Logger.w(TAG, "Screenshot capture returned null") + createFallbackScreenshot() + } + } catch (e: Exception) { + Logger.e(TAG, "Failed to capture screenshot", e) + createFallbackScreenshot() + } + } + + /** + * Creates a fallback black screenshot when capture fails. + */ + private fun createFallbackScreenshot(): Screenshot { + val bitmap = Bitmap.createBitmap(1080, 1920, Bitmap.Config.ARGB_8888) + bitmap.eraseColor(android.graphics.Color.BLACK) + + val base64Data = encodeBitmapToBase64(bitmap) + bitmap.recycle() + + return Screenshot( + base64Data = base64Data, + width = 1080, + height = 1920, + isSensitive = true + ) + } + + /** + * Encodes a Bitmap to a base64 string. + * + * @param bitmap The bitmap to encode + * @param format Compression format (default: WEBP_LOSSY) + * @param quality Compression quality 0-100 (default: 80) + * @return Base64-encoded string of the compressed image + */ + private fun encodeBitmapToBase64( + bitmap: Bitmap, + format: Bitmap.CompressFormat = Bitmap.CompressFormat.WEBP_LOSSY, + quality: Int = 80 + ): String { + val outputStream = ByteArrayOutputStream() + bitmap.compress(format, quality, outputStream) + return Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) + } + + companion object { + private const val TAG = "AccessibilityDeviceController" + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt new file mode 100644 index 0000000..66cd59e --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt @@ -0,0 +1,171 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.util.Point + +/** + * Device control mode enumeration. + * + * Defines the available methods for controlling the device. + * + * @property SHIZUKU Uses Shizuku shell commands (requires Shizuku app) + * @property ACCESSIBILITY Uses Android Accessibility Service + */ +enum class DeviceControlMode(val displayName: String) { + SHIZUKU("Shizuku"), + ACCESSIBILITY("无障碍服务") +} + +/** + * Abstract interface for device control operations. + * + * This interface defines the contract for device manipulation regardless of the + * underlying implementation (Shizuku shell commands or Accessibility Service). + * + * Implementations must handle: + * - Touch operations (tap, swipe, long press, double tap) + * - Key press events + * - App launching and info retrieval + * - Text input (method varies by implementation) + * - Screen capture + * - Permission management + * + * Requirements: Integration with both Shizuku and Accessibility modes + */ +interface IDeviceController { + + /** + * Gets the control mode of this device controller. + * + * @return The device control mode (SHIZUKU or ACCESSIBILITY) + */ + fun getMode(): DeviceControlMode + + /** + * Checks if the required permissions are granted. + * + * For Shizuku mode: Checks if Shizuku service is connected + * For Accessibility mode: Checks if accessibility service is enabled + * + * @return true if all required permissions are granted, false otherwise + */ + fun checkPermission(): Boolean + + /** + * Requests the required permissions from the user. + * + * For Shizuku mode: Launches Shizuku app or shows setup instructions + * For Accessibility mode: Opens system accessibility settings + * + * @param context Android context for starting intents + * @return true if the request was initiated successfully, false otherwise + */ + fun requestPermission(context: Context): Boolean + + // ==================== Touch Operations ==================== + + /** + * Performs a tap at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @return Result of the operation + */ + suspend fun tap(x: Int, y: Int): String + + /** + * Performs a double tap at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @return Result of the operation + */ + suspend fun doubleTap(x: Int, y: Int): String + + /** + * Performs a long press at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @param durationMs Duration of the long press in milliseconds + * @return Result of the operation + */ + suspend fun longPress(x: Int, y: Int, durationMs: Int = 3000): String + + /** + * Performs a swipe gesture using a list of points. + * + * @param points List of points defining the swipe path, must contain at least 2 points + * @param durationMs Total duration of the swipe in milliseconds + * @return Result of the operation + */ + suspend fun swipe(points: List, durationMs: Int): String + + // ==================== Key Press Operations ==================== + + /** + * Presses a key by its keycode. + * + * @param keyCode The Android KeyEvent keycode (e.g., KEYCODE_BACK, KEYCODE_HOME) + * @return Result of the operation + */ + suspend fun pressKey(keyCode: Int): String + + // ==================== App Operations ==================== + + /** + * Launches an app by its package name. + * + * @param packageName The package name of the app to launch + * @return Result of the operation + */ + suspend fun launchApp(packageName: String): String + + /** + * Gets the current foreground app's package name. + * + * @return The package name of the current foreground app, or empty string if not found + */ + suspend fun getCurrentApp(): String + + // ==================== Text Input ==================== + + /** + * Inputs text into the currently focused input field. + * + * Implementation varies by mode: + * - Shizuku: Uses keyboard switching via TextInputManager + * - Accessibility: Directly injects text via AccessibilityNodeInfo + * + * @param text The text to input + * @return Result of the operation + */ + suspend fun inputText(text: String): String + + // ==================== Screenshot ==================== + + /** + * Captures the current screen content. + * + * Implementation varies by mode: + * - Shizuku: Uses screencap shell command + * - Accessibility: Uses AccessibilityService.takeScreenshot() (Android 13+) + * + * @return Screenshot object containing the captured image data and metadata + */ + suspend fun captureScreen(): Screenshot + + // ==================== Constants ==================== + + companion object { + // Android KeyEvent keycodes + const val KEYCODE_BACK = 4 + const val KEYCODE_HOME = 3 + const val KEYCODE_VOLUME_UP = 24 + const val KEYCODE_VOLUME_DOWN = 25 + const val KEYCODE_POWER = 26 + const val KEYCODE_ENTER = 66 + const val KEYCODE_DEL = 67 + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt new file mode 100644 index 0000000..8ff5d5d --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt @@ -0,0 +1,101 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import com.kevinluo.autoglm.IUserService +import com.kevinluo.autoglm.input.TextInputManager +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.screenshot.ScreenshotService +import com.kevinluo.autoglm.util.Logger + +/** + * Device controller implementation using Shizuku shell commands. + * + * This controller uses the Shizuku framework to execute shell commands for device control. + * It requires the Shizuku app to be installed and the user service to be connected. + * + * @param userService Shizuku user service for executing shell commands + * @param textInputManager Manager for text input operations (keyboard switching) + * @param screenshotProvider Provider function for screenshot service + * + * Requirements: Shizuku-based device control + */ +class ShizukuDeviceController( + private val userService: IUserService, + private val textInputManager: TextInputManager, + private val screenshotProvider: () -> ScreenshotService +) : IDeviceController { + + private val executor = DeviceExecutor(userService) + + override fun getMode(): DeviceControlMode = DeviceControlMode.SHIZUKU + + override fun checkPermission(): Boolean { + return try { + // Try to execute a simple command to check if Shizuku is connected + val result = userService.executeCommand("echo test") + result.contains("test") + } catch (e: Exception) { + Logger.w(TAG, "Shizuku permission check failed: ${e.message}") + false + } + } + + override fun requestPermission(context: Context): Boolean { + try { + // Launch Shizuku app + val intent = context.packageManager.getLaunchIntentForPackage("moe.shizuku.privileged.api") + if (intent != null) { + context.startActivity(intent) + return true + } + } catch (e: Exception) { + Logger.e(TAG, "Failed to launch Shizuku app", e) + } + return false + } + + override suspend fun tap(x: Int, y: Int): String { + return executor.tap(x, y) + } + + override suspend fun doubleTap(x: Int, y: Int): String { + return executor.doubleTap(x, y) + } + + override suspend fun longPress(x: Int, y: Int, durationMs: Int): String { + return executor.longPress(x, y, durationMs) + } + + override suspend fun swipe(points: List, durationMs: Int): String { + return executor.swipe(points, durationMs) + } + + override suspend fun pressKey(keyCode: Int): String { + return executor.pressKey(keyCode) + } + + override suspend fun launchApp(packageName: String): String { + return executor.launchApp(packageName) + } + + override suspend fun getCurrentApp(): String { + return executor.getCurrentApp() + } + + override suspend fun inputText(text: String): String { + val result = textInputManager.typeText(text) + return if (result.success) { + "Text input successful: ${text.take(30)}..." + } else { + "Text input failed: ${result.message}" + } + } + + override suspend fun captureScreen(): Screenshot { + return screenshotProvider().capture() + } + + companion object { + private const val TAG = "ShizukuDeviceController" + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt index 0dafc97..b15f616 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt @@ -10,6 +10,7 @@ import android.widget.ImageButton import android.widget.ImageView import android.widget.LinearLayout import android.widget.TextView +import androidx.activity.OnBackPressedCallback import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity import androidx.core.content.ContextCompat @@ -39,29 +40,42 @@ import java.util.Locale * */ class HistoryActivity : AppCompatActivity() { - + private lateinit var historyManager: HistoryManager private lateinit var recyclerView: RecyclerView private lateinit var emptyState: LinearLayout private lateinit var adapter: HistoryAdapter - + // Multi-select mode private lateinit var normalToolbar: LinearLayout private lateinit var selectionToolbar: LinearLayout private lateinit var selectionCountText: TextView private var isSelectionMode = false - + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_history) - + historyManager = HistoryManager.getInstance(this) - + Logger.d(TAG, "HistoryActivity created") setupViews() observeHistory() + + // Handle back press using modern OnBackPressedDispatcher + val backCallback = object : OnBackPressedCallback(true) { + override fun handleOnBackPressed() { + if (isSelectionMode) { + exitSelectionMode() + } else { + isEnabled = false + onBackPressedDispatcher.onBackPressed() + } + } + } + onBackPressedDispatcher.addCallback(this, backCallback) } - + /** * Sets up all view references and click listeners. */ @@ -69,31 +83,31 @@ class HistoryActivity : AppCompatActivity() { normalToolbar = findViewById(R.id.normalToolbar) selectionToolbar = findViewById(R.id.selectionToolbar) selectionCountText = findViewById(R.id.selectionCountText) - + findViewById(R.id.backBtn).setOnClickListener { finish() } - + findViewById(R.id.clearAllBtn).setOnClickListener { showClearAllDialog() } - + // Selection toolbar buttons findViewById(R.id.cancelSelectionBtn).setOnClickListener { exitSelectionMode() } - + findViewById(R.id.selectAllBtn).setOnClickListener { adapter.selectAll() } - + findViewById(R.id.deleteSelectedBtn).setOnClickListener { showDeleteSelectedDialog() } - + recyclerView = findViewById(R.id.historyRecyclerView) emptyState = findViewById(R.id.emptyState) - + adapter = HistoryAdapter( onItemClick = { task -> if (isSelectionMode) { @@ -115,11 +129,11 @@ class HistoryActivity : AppCompatActivity() { } } ) - + recyclerView.layoutManager = LinearLayoutManager(this) recyclerView.adapter = adapter } - + /** * Observes the history list and updates the UI accordingly. */ @@ -129,7 +143,7 @@ class HistoryActivity : AppCompatActivity() { adapter.submitList(list) emptyState.visibility = if (list.isEmpty()) View.VISIBLE else View.GONE recyclerView.visibility = if (list.isEmpty()) View.GONE else View.VISIBLE - + // Exit selection mode if list becomes empty if (list.isEmpty() && isSelectionMode) { exitSelectionMode() @@ -137,7 +151,7 @@ class HistoryActivity : AppCompatActivity() { } } } - + /** * Enters multi-select mode for batch operations. */ @@ -148,7 +162,7 @@ class HistoryActivity : AppCompatActivity() { selectionToolbar.visibility = View.VISIBLE Logger.d(TAG, "Entered selection mode") } - + /** * Exits multi-select mode and clears selection. */ @@ -160,7 +174,7 @@ class HistoryActivity : AppCompatActivity() { selectionToolbar.visibility = View.GONE Logger.d(TAG, "Exited selection mode") } - + /** * Updates the selection count display in the toolbar. * @@ -169,7 +183,7 @@ class HistoryActivity : AppCompatActivity() { private fun updateSelectionCount(count: Int) { selectionCountText.text = getString(R.string.history_selected_count, count) } - + /** * Opens the task detail activity for the given task. * @@ -181,14 +195,14 @@ class HistoryActivity : AppCompatActivity() { intent.putExtra(HistoryDetailActivity.EXTRA_TASK_ID, task.id) startActivity(intent) } - + /** * Shows a confirmation dialog for deleting selected tasks. */ private fun showDeleteSelectedDialog() { val selectedIds = adapter.getSelectedIds() if (selectedIds.isEmpty()) return - + AlertDialog.Builder(this) .setTitle(R.string.history_delete_selected) .setMessage(getString(R.string.history_delete_selected_confirm, selectedIds.size)) @@ -201,7 +215,7 @@ class HistoryActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + /** * Shows a confirmation dialog for clearing all history. */ @@ -218,16 +232,7 @@ class HistoryActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - - @Deprecated("Deprecated in Java") - override fun onBackPressed() { - if (isSelectionMode) { - exitSelectionMode() - } else { - super.onBackPressed() - } - } - + companion object { private const val TAG = "HistoryActivity" } @@ -250,12 +255,12 @@ class HistoryAdapter( private val onItemLongClick: (TaskHistory) -> Unit, private val onSelectionChanged: (Int) -> Unit ) : RecyclerView.Adapter() { - + private var items: List = emptyList() private val selectedIds = mutableSetOf() private var isSelectionMode = false private val dateFormat = SimpleDateFormat("MM-dd HH:mm", Locale.getDefault()) - + /** * Submits a new list of items to display. * @@ -267,7 +272,7 @@ class HistoryAdapter( selectedIds.retainAll(list.map { it.id }.toSet()) notifyDataSetChanged() } - + /** * Enables or disables selection mode. * @@ -277,7 +282,7 @@ class HistoryAdapter( isSelectionMode = enabled notifyDataSetChanged() } - + /** * Toggles the selection state of a task. * @@ -292,7 +297,7 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(selectedIds.size) } - + /** * Selects all items in the list. */ @@ -302,7 +307,7 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(selectedIds.size) } - + /** * Clears all selections. */ @@ -311,26 +316,26 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(0) } - + /** * Gets the set of currently selected task IDs. * * @return Immutable copy of selected task IDs */ fun getSelectedIds(): Set = selectedIds.toSet() - + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { val view = LayoutInflater.from(parent.context) .inflate(R.layout.item_history_task, parent, false) return ViewHolder(view) } - + override fun onBindViewHolder(holder: ViewHolder, position: Int) { holder.bind(items[position]) } - + override fun getItemCount(): Int = items.size - + /** * ViewHolder for history list items. * @@ -343,7 +348,7 @@ class HistoryAdapter( private val timeText: TextView = itemView.findViewById(R.id.timeText) private val stepsText: TextView = itemView.findViewById(R.id.stepsText) private val durationText: TextView = itemView.findViewById(R.id.durationText) - + /** * Binds task data to the view. * @@ -357,7 +362,7 @@ class HistoryAdapter( R.string.history_duration_format, formatDuration(task.duration) ) - + if (task.success) { statusIcon.setImageResource(R.drawable.ic_check_circle) statusIcon.setColorFilter( @@ -369,22 +374,22 @@ class HistoryAdapter( ContextCompat.getColor(itemView.context, R.color.status_error) ) } - + // Handle selection mode checkBox.visibility = if (isSelectionMode) View.VISIBLE else View.GONE checkBox.isChecked = selectedIds.contains(task.id) - + checkBox.setOnClickListener { toggleSelection(task.id) } - + itemView.setOnClickListener { onItemClick(task) } itemView.setOnLongClickListener { onItemLongClick(task) true } } - + /** * Formats duration in milliseconds to a human-readable string. * diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt index a2deb84..3bba2d7 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt @@ -569,12 +569,16 @@ class HistoryDetailActivity : AppCompatActivity() { FileOutputStream(file).use { out -> bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) } - - // Notify gallery - val intent = Intent(Intent.ACTION_MEDIA_SCANNER_SCAN_FILE) - intent.data = android.net.Uri.fromFile(file) - sendBroadcast(intent) - + + // Notify gallery using MediaScannerConnection (recommended approach) + android.media.MediaScannerConnection.scanFile( + this, + arrayOf(file.absolutePath), + arrayOf("image/webp") + ) { path, uri -> + Logger.d(TAG, "Scanned $path: $uri") + } + true } } diff --git a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt index 40f04f7..ffd6fc2 100644 --- a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt @@ -78,10 +78,20 @@ interface FloatingWindowController { * */ class ScreenshotService( - private val userService: IUserService, + userService: IUserService? = null, + private val screenshotProvider: (suspend () -> Screenshot)? = null, private val floatingWindowControllerProvider: () -> FloatingWindowController? = { null } ) { - + // Store userService for internal use (only used for Shizuku mode via shell commands) + private val userService: IUserService? = userService + + init { + // At least one capture method must be available + require(userService != null || screenshotProvider != null) { + "Either userService or screenshotProvider must be provided" + } + } + companion object { private const val TAG = "ScreenshotService" private const val HIDE_DELAY_MS = 200L @@ -156,31 +166,43 @@ class ScreenshotService( * */ private suspend fun captureScreen(): Screenshot = withContext(Dispatchers.IO) { + // Use screenshotProvider if available (Accessibility mode) + if (screenshotProvider != null) { + Logger.d(TAG, "Using screenshot provider (Accessibility mode)") + return@withContext try { + screenshotProvider.invoke() + } catch (e: Exception) { + Logger.e(TAG, "Screenshot provider failed, using fallback", e) + createFallbackScreenshot() + } + } + + // Otherwise use shell command method (Shizuku mode) try { - Logger.d(TAG, "Executing screencap command") - + Logger.d(TAG, "Executing screencap command (Shizuku mode)") + val pngData = executeScreencapToBytes() - + if (pngData == null || pngData.isEmpty()) { Logger.w(TAG, "Failed to capture screenshot, returning fallback") return@withContext createFallbackScreenshot() } - + Logger.d(TAG, "PNG data captured: ${pngData.size} bytes") - + // Decode PNG to bitmap var bitmap = BitmapFactory.decodeByteArray(pngData, 0, pngData.size) if (bitmap == null) { Logger.w(TAG, "Failed to decode PNG, returning fallback") return@withContext createFallbackScreenshot() } - + val originalWidth = bitmap.width val originalHeight = bitmap.height - + // Calculate scaled dimensions based on max size constraints val (scaledWidth, scaledHeight) = calculateOptimalDimensions(originalWidth, originalHeight) - + // Scale bitmap if needed if (scaledWidth != originalWidth || scaledHeight != originalHeight) { val scaledBitmap = Bitmap.createScaledBitmap(bitmap, scaledWidth, scaledHeight, true) @@ -188,7 +210,7 @@ class ScreenshotService( bitmap = scaledBitmap Logger.d(TAG, "Scaled from ${originalWidth}x${originalHeight} to ${scaledWidth}x${scaledHeight}") } - + // Convert to WebP for better compression val webpStream = ByteArrayOutputStream() bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, WEBP_QUALITY, webpStream) @@ -257,16 +279,21 @@ class ScreenshotService( * */ private suspend fun executeScreencapToBytes(): ByteArray? = coroutineScope { + val service = userService ?: run { + Logger.w(TAG, "UserService not available for shell command screenshot") + return@coroutineScope null + } + val timestamp = System.currentTimeMillis() val pngFile = "/data/local/tmp/screenshot_$timestamp.png" val base64File = "$pngFile.b64" - + try { Logger.d(TAG, "Attempting screenshot capture") val startTime = System.currentTimeMillis() - + // Capture screenshot and pipe to base64 - val captureResult = userService.executeCommand( + val captureResult = service.executeCommand( "screencap -p | base64 > $base64File && stat -c %s $base64File" ) diff --git a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt index 1f14a08..7d171ef 100644 --- a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt @@ -22,6 +22,7 @@ import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView import com.kevinluo.autoglm.R import com.kevinluo.autoglm.agent.AgentConfig +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.model.ModelClient import com.kevinluo.autoglm.model.ModelConfig import com.kevinluo.autoglm.util.LogFileManager @@ -41,7 +42,12 @@ import kotlinx.coroutines.launch class SettingsActivity : AppCompatActivity() { private lateinit var settingsManager: SettingsManager - + + // Device control mode views + private lateinit var controlModeRadioGroup: RadioGroup + private lateinit var controlModeShizuku: RadioButton + private lateinit var controlModeAccessibility: RadioButton + // Profile selector views private lateinit var profileSelectorLayout: TextInputLayout private lateinit var profileSelector: AutoCompleteTextView @@ -115,6 +121,11 @@ class SettingsActivity : AppCompatActivity() { * Initializes all view references. */ private fun initViews() { + // Device control mode + controlModeRadioGroup = findViewById(R.id.controlModeRadioGroup) + controlModeShizuku = findViewById(R.id.controlModeShizuku) + controlModeAccessibility = findViewById(R.id.controlModeAccessibility) + // Profile selector profileSelectorLayout = findViewById(R.id.profileSelectorLayout) profileSelector = findViewById(R.id.profileSelector) @@ -177,7 +188,14 @@ class SettingsActivity : AppCompatActivity() { Logger.d(TAG, "Loading current settings") val modelConfig = settingsManager.getModelConfig() val agentConfig = settingsManager.getAgentConfig() - + val controlMode = settingsManager.getDeviceControlMode() + + // Load device control mode + when (controlMode) { + DeviceControlMode.SHIZUKU -> controlModeShizuku.isChecked = true + DeviceControlMode.ACCESSIBILITY -> controlModeAccessibility.isChecked = true + } + // Load saved profiles loadSavedProfiles() @@ -316,6 +334,16 @@ class SettingsActivity : AppCompatActivity() { modelNameInput.setOnFocusChangeListener { _, _ -> modelNameLayout.error = null } maxStepsInput.setOnFocusChangeListener { _, _ -> maxStepsLayout.error = null } screenshotDelayInput.setOnFocusChangeListener { _, _ -> screenshotDelayLayout.error = null } + + // Device control mode change listener + controlModeRadioGroup.setOnCheckedChangeListener { _, checkedId -> + val newMode = when (checkedId) { + R.id.controlModeShizuku -> DeviceControlMode.SHIZUKU + R.id.controlModeAccessibility -> DeviceControlMode.ACCESSIBILITY + else -> return@setOnCheckedChangeListener + } + Logger.d(TAG, "Device control mode changed to: ${newMode.name}") + } } /** @@ -585,14 +613,22 @@ class SettingsActivity : AppCompatActivity() { Logger.i(TAG, "Saving settings") val baseUrl = baseUrlInput.text?.toString()?.trim() ?: "" val modelName = modelNameInput.text?.toString()?.trim() ?: "" - val apiKey = apiKeyInput.text?.toString()?.trim().let { - if (it.isNullOrEmpty()) "EMPTY" else it + val apiKey = apiKeyInput.text?.toString()?.trim().let { + if (it.isNullOrEmpty()) "EMPTY" else it } val maxSteps = maxStepsInput.text?.toString()?.trim()?.toIntOrNull() ?: 100 val screenshotDelaySeconds = screenshotDelayInput.text?.toString()?.trim()?.toDoubleOrNull() ?: 2.0 val screenshotDelayMs = (screenshotDelaySeconds * 1000).toLong() val language = if (languageEnglish.isChecked) "en" else "cn" - + + // Save device control mode + val controlMode = when (controlModeRadioGroup.checkedRadioButtonId) { + R.id.controlModeShizuku -> DeviceControlMode.SHIZUKU + R.id.controlModeAccessibility -> DeviceControlMode.ACCESSIBILITY + else -> DeviceControlMode.SHIZUKU + } + settingsManager.saveDeviceControlMode(controlMode) + // Create and save model config val modelConfig = ModelConfig( baseUrl = baseUrl, @@ -600,7 +636,7 @@ class SettingsActivity : AppCompatActivity() { modelName = modelName ) settingsManager.saveModelConfig(modelConfig) - + // Create and save agent config val agentConfig = AgentConfig( maxSteps = maxSteps, @@ -608,7 +644,7 @@ class SettingsActivity : AppCompatActivity() { screenshotDelayMs = screenshotDelayMs ) settingsManager.saveAgentConfig(agentConfig) - + Toast.makeText(this, R.string.settings_saved, Toast.LENGTH_SHORT).show() finish() } diff --git a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt index 9c8dc5e..545ffd0 100644 --- a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt @@ -6,6 +6,7 @@ import android.os.Build import androidx.security.crypto.EncryptedSharedPreferences import androidx.security.crypto.MasterKey import com.kevinluo.autoglm.agent.AgentConfig +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.model.ModelConfig import com.kevinluo.autoglm.util.Logger import org.json.JSONArray @@ -95,7 +96,10 @@ class SettingsManager(private val context: Context) { // Dev profiles import key private const val KEY_DEV_PROFILES_IMPORTED = "dev_profiles_imported" - + + // Device control mode key + private const val KEY_DEVICE_CONTROL_MODE = "device_control_mode" + // Default values private val DEFAULT_MODEL_CONFIG = ModelConfig() private val DEFAULT_AGENT_CONFIG = AgentConfig() @@ -104,6 +108,7 @@ class SettingsManager(private val context: Context) { // Cache for detecting config changes private var lastModelConfig: ModelConfig? = null private var lastAgentConfig: AgentConfig? = null + private var lastDeviceControlMode: DeviceControlMode? = null // Regular preferences for non-sensitive data private val prefs: SharedPreferences by lazy { @@ -285,13 +290,17 @@ class SettingsManager(private val context: Context) { fun hasConfigChanged(): Boolean { val currentModelConfig = getModelConfig() val currentAgentConfig = getAgentConfig() - - val changed = lastModelConfig != currentModelConfig || lastAgentConfig != currentAgentConfig - + val currentDeviceControlMode = getDeviceControlMode() + + val changed = lastModelConfig != currentModelConfig || + lastAgentConfig != currentAgentConfig || + lastDeviceControlMode != currentDeviceControlMode + // Update cache lastModelConfig = currentModelConfig lastAgentConfig = currentAgentConfig - + lastDeviceControlMode = currentDeviceControlMode + return changed } @@ -689,4 +698,42 @@ class SettingsManager(private val context: Context) { -1 } } + + // ==================== Device Control Mode ==================== + + /** + * Gets the current device control mode. + * + * Returns SHIZUKU as default if not previously set. + * + * @return The current device control mode + */ + fun getDeviceControlMode(): DeviceControlMode { + val modeName = prefs.getString(KEY_DEVICE_CONTROL_MODE, DeviceControlMode.SHIZUKU.name) + return try { + DeviceControlMode.valueOf(modeName ?: DeviceControlMode.SHIZUKU.name) + } catch (e: Exception) { + Logger.w(TAG, "Invalid device control mode: $modeName, using SHIZUKU") + DeviceControlMode.SHIZUKU + } + } + + /** + * Saves the device control mode. + * + * @param mode The device control mode to save + */ + fun saveDeviceControlMode(mode: DeviceControlMode) { + Logger.d(TAG, "Saving device control mode: ${mode.name}") + prefs.edit().putString(KEY_DEVICE_CONTROL_MODE, mode.name).apply() + } + + /** + * Gets the display name of the current device control mode. + * + * @return The display name (e.g., "Shizuku" or "无障碍服务") + */ + fun getDeviceControlModeDisplayName(): String { + return getDeviceControlMode().displayName + } } diff --git a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt index 2f1fd1d..c236879 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt @@ -775,7 +775,9 @@ class FloatingWindowService : Service(), FloatingWindowController { gravity = Gravity.TOP or Gravity.START x = 0 y = 0 - // Allow keyboard input + // Allow keyboard input - Note: SOFT_INPUT_ADJUST_RESIZE is deprecated but still + // required for floating windows to resize when keyboard is shown + @Suppress("DEPRECATION") softInputMode = WindowManager.LayoutParams.SOFT_INPUT_ADJUST_RESIZE } } diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index ec9eaff..6ffee13 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -59,14 +59,20 @@ android:paddingHorizontal="16dp" android:paddingBottom="32dp"> - - + + android:layout_marginBottom="12dp"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AutoGLM For Android - + Shizuku 状态 未知 @@ -10,26 +10,26 @@ 已连接 授权 打开 Shizuku - + 悬浮窗权限 已授权 未授权 授权 - + 输入法 已启用 未启用 去启用 需要启用 AutoGLM Keyboard 才能输入文字 - + 任务输入 描述你想要执行的任务... 启动 停止 - + 状态 空闲 @@ -42,12 +42,12 @@ 步骤: %d 暂停 继续 - + 执行日志 查看任务执行的详细日志 日志将显示在这里... - + 请先启动 Shizuku Shizuku 版本过低 @@ -61,7 +61,7 @@ 任务已开始 任务已取消 需要悬浮窗权限 - + 设置 模型配置 @@ -108,7 +108,7 @@ 连接失败: %s 连接超时: %s 请先填写完整的配置信息 - + 任务模板 保存常用任务，快速填入 @@ -124,7 +124,7 @@ 确定要删除此模板吗？ 暂无模板 选择模板 - + 高级设置 自定义系统提示词 @@ -139,7 +139,7 @@ 已重置为默认提示词 已自定义 默认 - + 确认操作 确定要执行此操作吗？ @@ -147,13 +147,13 @@ 取消 选择选项 请选择以下选项之一 - + 需要手动操作 请完成以下操作 操作说明将显示在这里 已完成，继续 - + 思考 操作 @@ -164,15 +164,15 @@ 等待确认 打开悬浮窗 新任务 - + 悬浮窗 历史记录 - + AutoGLM 显示/隐藏悬浮窗 - + 历史记录 任务详情 @@ -204,7 +204,7 @@ 提示词已复制到剪贴板 加载更多 加载更多 (剩余 %d 步) - + 调试日志 导出日志用于问题排查 @@ -215,11 +215,28 @@ 导出失败 暂无日志 确定要清空所有日志吗？ - + AutoGLM Keyboard 就绪 文本输入由 AutoGLM 控制 AutoGLM 键盘 已启用 + + + AutoGLM 无障碍服务，用于辅助残障人士进行设备自动化控制。 + 无障碍权限 + 已开启 + 未开启 + 去开启 + 无障碍模式不支持 Android 13 以下版本 + + + 设备控制模式 + 选择控制设备的方式 + Shizuku（推荐） + 无障碍服务 + 需要安装 Shizuku 应用，功能完整，兼容性好 + 无需安装额外应用，需要 Android 13+ + 控制模式已更改，请重启应用以生效 diff --git a/app/src/main/res/xml/accessibility_service_config.xml b/app/src/main/res/xml/accessibility_service_config.xml new file mode 100644 index 0000000..37e7efd --- /dev/null +++ b/app/src/main/res/xml/accessibility_service_config.xml @@ -0,0 +1,25 @@ + + + + From 5047a511bb4ba5eba5fa04b60ad0464db9b0ac25 Mon Sep 17 00:00:00 2001 From: Repobor Date: Sun, 28 Dec 2025 20:20:54 +0800 Subject: [PATCH 2/6] fix(ui): Enable start button in accessibility mode on initial load **Problem:** Start button was disabled in accessibility mode because: 1. isServiceConnected only checked userService != null 2. In accessibility mode, userService is never set 3. Components weren't initialized in onCreate() for accessibility mode 4. onResume() couldn't initialize because of circular dependency **Solution:** 1. Modified isServiceConnected to check mode-specific conditions: - SHIZUKU: checks userService != null - ACCESSIBILITY: checks _phoneAgent != null 2. Added initialization in onCreate() for accessibility mode 3. Simplified onResume() logic to always check for config changes 4. Added updateTaskButtonStates() call in onCreate() **Testing:** - Switch to accessibility mode in settings - Enable accessibility service - Start button should now be clickable --- .../java/com/kevinluo/autoglm/ComponentManager.kt | 13 ++++++++++--- .../main/java/com/kevinluo/autoglm/MainActivity.kt | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt index 2ddcef7..687d2b5 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt @@ -21,7 +21,6 @@ import com.kevinluo.autoglm.screenshot.ScreenshotService import com.kevinluo.autoglm.settings.SettingsManager import com.kevinluo.autoglm.ui.FloatingWindowService import com.kevinluo.autoglm.util.HumanizedSwipeGenerator -import com.kevinluo.autoglm.util.Logger /** * Centralized component manager for dependency injection and lifecycle management. @@ -95,10 +94,18 @@ class ComponentManager private constructor(private val context: Context) { private var _swipeGenerator: HumanizedSwipeGenerator? = null /** - * Checks if the UserService is connected. + * Checks if the device controller is ready. + * For Shizuku mode: checks if Shizuku service is connected + * For Accessibility mode: checks if PhoneAgent is initialized */ val isServiceConnected: Boolean - get() = userService != null + get() { + val controlMode = settingsManager.getDeviceControlMode() + return when (controlMode) { + DeviceControlMode.SHIZUKU -> userService != null + DeviceControlMode.ACCESSIBILITY -> _phoneAgent != null + } + } /** * Gets the DeviceExecutor instance. diff --git a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt index 5ba9594..cb2d949 100644 --- a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt @@ -211,6 +211,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { updateKeyboardStatus() updateAccessibilityStatus() updateTaskStatus(TaskStatus.IDLE) + updateTaskButtonStates() } /** @@ -381,8 +382,8 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } componentManager.setPhoneAgentListener(this) setupConfirmationCallback() - updateTaskButtonStates() } + updateTaskButtonStates() } override fun onDestroy() { From c8ffa3b04bd0e99489f92c64b5d5982e603d3d3c Mon Sep 17 00:00:00 2001 From: Repobor Date: Sun, 28 Dec 2025 20:27:34 +0800 Subject: [PATCH 3/6] fix(screenshot): Fix ScreenshotService to use correct implementation for each mode **Problem:** In accessibility mode, ScreenshotService was still trying to use Shizuku's shell commands instead of using AccessibilityDeviceController's captureScreen() method. **Root Cause:** Incorrect ScreenshotService constructor calls in both modes: 1. Accessibility mode: OLD: ScreenshotService(stubUserService) { FloatingWindowService.getInstance() } - userService = stubUserService (wrong, should be null) - screenshotProvider = { FloatingWindowService.getInstance() } (wrong type, should call captureScreen()) - floatingWindowControllerProvider = default (missing) 2. Shizuku mode: OLD: ScreenshotService(service) { FloatingWindowService.getInstance() } - userService = service (correct) - screenshotProvider = { FloatingWindowService.getInstance() } (wrong, should be null) - floatingWindowControllerProvider = default (missing) **Solution:** Use named parameters with correct types: 1. Accessibility mode: ScreenshotService( userService = null, screenshotProvider = { _deviceController!!.captureScreen() }, floatingWindowControllerProvider = { FloatingWindowService.getInstance() } ) 2. Shizuku mode: ScreenshotService( userService = service, screenshotProvider = null, floatingWindowControllerProvider = { FloatingWindowService.getInstance() } ) Now: - Accessibility mode uses AccessibilityDeviceController.captureScreen() - Shizuku mode uses shell commands via userService - Both modes properly hide/show floating window --- .../java/com/kevinluo/autoglm/ComponentManager.kt | 15 ++++++++++++--- .../accessibility/AutoGLMAccessibilityService.kt | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt index 687d2b5..1b16cf7 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt @@ -221,7 +221,11 @@ class ComponentManager private constructor(private val context: Context) { // Create ScreenshotService with floating window controller provider // Use a provider function so it can get the current instance dynamically - _screenshotService = ScreenshotService(service) { FloatingWindowService.getInstance() } + _screenshotService = ScreenshotService( + userService = service, + screenshotProvider = null, // For Shizuku mode, use shell commands + floatingWindowControllerProvider = { FloatingWindowService.getInstance() } + ) // Create ActionHandler with floating window provider to hide window during touch operations _actionHandler = ActionHandler( @@ -266,8 +270,13 @@ class ComponentManager private constructor(private val context: Context) { // Create TextInputManager with stub service (won't be actually used) _textInputManager = TextInputManager(stubUserService) - // Create ScreenshotService for accessibility mode - _screenshotService = ScreenshotService(stubUserService) { FloatingWindowService.getInstance() } + // Create ScreenshotService for accessibility mode with proper screenshotProvider + // The screenshotProvider delegates to AccessibilityDeviceController for screenshots + _screenshotService = ScreenshotService( + userService = null, // No userService needed in accessibility mode + screenshotProvider = { _deviceController!!.captureScreen() }, + floatingWindowControllerProvider = { FloatingWindowService.getInstance() } + ) // Create ActionHandler for accessibility mode // The DeviceExecutor and TextInputManager instances exist but their actual methods diff --git a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt index 04b9b9b..a24c4bd 100644 --- a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt @@ -103,8 +103,8 @@ class AutoGLMAccessibilityService : AccessibilityService() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { try { takeScreenshot(/* displayId= */ 0, /* executor= */ mainExecutor, - object : android.accessibilityservice.AccessibilityService.TakeScreenshotCallback { - override fun onSuccess(screenshot: android.accessibilityservice.AccessibilityService.ScreenshotResult) { + object : TakeScreenshotCallback { + override fun onSuccess(screenshot: ScreenshotResult) { try { val hardwareBuffer = screenshot.hardwareBuffer val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, screenshot.colorSpace) From a98dcd424f37f424c298f2255ecaa4df4c374562 Mon Sep 17 00:00:00 2001 From: Repobor Date: Mon, 29 Dec 2025 20:52:22 +0800 Subject: [PATCH 4/6] fix(accessibility): secure issus. --- .../com/kevinluo/autoglm/ComponentManager.kt | 35 ++++------ .../AutoGLMAccessibilityService.kt | 14 +++- .../kevinluo/autoglm/action/ActionHandler.kt | 68 ++++++++++++------- .../device/AccessibilityDeviceController.kt | 11 ++- .../res/xml/accessibility_service_config.xml | 1 + 5 files changed, 75 insertions(+), 54 deletions(-) diff --git a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt index 1b16cf7..354f3ab 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt @@ -227,12 +227,18 @@ class ComponentManager private constructor(private val context: Context) { floatingWindowControllerProvider = { FloatingWindowService.getInstance() } ) - // Create ActionHandler with floating window provider to hide window during touch operations + // Create ShizukuDeviceController for Shizuku mode + _deviceController = ShizukuDeviceController( + userService = service, + textInputManager = _textInputManager!!, + screenshotProvider = { _screenshotService!! } + ) + + // Create ActionHandler with IDeviceController _actionHandler = ActionHandler( - deviceExecutor = _deviceExecutor!!, + deviceController = _deviceController!!, appResolver = appResolver, swipeGenerator = swipeGenerator, - textInputManager = _textInputManager!!, floatingWindowProvider = { FloatingWindowService.getInstance() } ) @@ -251,25 +257,12 @@ class ComponentManager private constructor(private val context: Context) { /** * Initializes components for accessibility mode. - * Creates a stub device controller when Accessibility Service is available. + * Creates a device controller when Accessibility Service is available. */ private fun initializeAccessibilityComponents() { // Create AccessibilityDeviceController _deviceController = AccessibilityDeviceController(context) - // Create a stub UserService for accessibility mode - val stubUserService = object : IUserService { - override fun executeCommand(cmd: String): String = "" - override fun destroy() {} - override fun asBinder() = null - } - - // Create DeviceExecutor with stub service (won't be actually used) - _deviceExecutor = DeviceExecutor(stubUserService) - - // Create TextInputManager with stub service (won't be actually used) - _textInputManager = TextInputManager(stubUserService) - // Create ScreenshotService for accessibility mode with proper screenshotProvider // The screenshotProvider delegates to AccessibilityDeviceController for screenshots _screenshotService = ScreenshotService( @@ -278,14 +271,12 @@ class ComponentManager private constructor(private val context: Context) { floatingWindowControllerProvider = { FloatingWindowService.getInstance() } ) - // Create ActionHandler for accessibility mode - // The DeviceExecutor and TextInputManager instances exist but their actual methods - // are not used in accessibility mode - all operations go through IDeviceController + // Create ActionHandler with IDeviceController (AccessibilityDeviceController) + // No need for DeviceExecutor or TextInputManager, all operations go through the controller _actionHandler = ActionHandler( - deviceExecutor = _deviceExecutor!!, + deviceController = _deviceController!!, appResolver = appResolver, swipeGenerator = swipeGenerator, - textInputManager = _textInputManager!!, floatingWindowProvider = { FloatingWindowService.getInstance() } ) diff --git a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt index a24c4bd..f776a68 100644 --- a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt @@ -102,15 +102,19 @@ class AutoGLMAccessibilityService : AccessibilityService() { suspend fun captureScreen(): Bitmap? = suspendCancellableCoroutine { continuation -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { try { + Logger.d(TAG, "Starting screenshot capture via AccessibilityService.takeScreenshot()") takeScreenshot(/* displayId= */ 0, /* executor= */ mainExecutor, object : TakeScreenshotCallback { override fun onSuccess(screenshot: ScreenshotResult) { try { + Logger.d(TAG, "Screenshot callback: onSuccess") val hardwareBuffer = screenshot.hardwareBuffer val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, screenshot.colorSpace) if (bitmap != null) { + Logger.d(TAG, "Successfully wrapped HardwareBuffer to Bitmap: ${bitmap.width}x${bitmap.height}") continuation.resume(bitmap) } else { + Logger.w(TAG, "Failed to wrap HardwareBuffer to Bitmap") continuation.resume(null) } hardwareBuffer.close() @@ -121,16 +125,20 @@ class AutoGLMAccessibilityService : AccessibilityService() { } override fun onFailure(error: Int) { - Logger.e(TAG, "Screenshot capture failed with error code: $error") + Logger.e(TAG, "Screenshot callback: onFailure with error code: $error") continuation.resume(null) } }) + } catch (e: SecurityException) { + Logger.e(TAG, "SecurityException: Service doesn't have screenshot capability. " + + "Please ensure: 1) Accessibility service is enabled 2) accessibility_service_config.xml has canTakeScreenshot=true 3) Device is Android 13+", e) + continuation.resume(null) } catch (e: Exception) { - Logger.e(TAG, "Failed to capture screenshot", e) + Logger.e(TAG, "Failed to call takeScreenshot", e) continuation.resume(null) } } else { - Logger.w(TAG, "Screenshot not supported on Android < 13") + Logger.w(TAG, "Screenshot not supported on Android < 13 (current: ${Build.VERSION.SDK_INT})") continuation.resume(null) } } diff --git a/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt b/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt index 100cda2..6ee5c5b 100644 --- a/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt +++ b/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt @@ -1,8 +1,7 @@ package com.kevinluo.autoglm.action import com.kevinluo.autoglm.app.AppResolver -import com.kevinluo.autoglm.device.DeviceExecutor -import com.kevinluo.autoglm.input.TextInputManager +import com.kevinluo.autoglm.device.IDeviceController import com.kevinluo.autoglm.screenshot.FloatingWindowController import com.kevinluo.autoglm.util.CoordinateConverter import com.kevinluo.autoglm.util.ErrorHandler @@ -11,25 +10,23 @@ import com.kevinluo.autoglm.util.Logger import kotlinx.coroutines.delay /** - * Handles execution of agent actions by coordinating with DeviceExecutor, - * AppResolver, HumanizedSwipeGenerator, and TextInputManager. + * Handles execution of agent actions by coordinating with IDeviceController, + * AppResolver, and HumanizedSwipeGenerator. * * This class is responsible for translating high-level [AgentAction] commands * into device-level operations. It manages floating window visibility during * touch operations to prevent interference. * - * @param deviceExecutor Executor for device-level operations (tap, swipe, etc.) + * @param deviceController Device controller for device-level operations (tap, swipe, text input, etc.) * @param appResolver Resolver for app name to package name mapping * @param swipeGenerator Generator for humanized swipe paths - * @param textInputManager Manager for text input operations * @param floatingWindowProvider Optional provider for floating window controller * */ class ActionHandler( - private val deviceExecutor: DeviceExecutor, + private val deviceController: IDeviceController, private val appResolver: AppResolver, private val swipeGenerator: HumanizedSwipeGenerator, - private val textInputManager: TextInputManager, private val floatingWindowProvider: (() -> FloatingWindowController?)? = null ) { @@ -97,6 +94,26 @@ class ActionHandler( floatingWindowProvider?.invoke()?.show() } + /** + * Handles text input via the device controller. + * + * @param text The text to input + * @return ActionResult with success status and message + */ + private suspend fun inputText(text: String): ActionResult { + return try { + val result = deviceController.inputText(text) + if (result.contains("Error", ignoreCase = true)) { + ActionResult(false, false, result) + } else { + ActionResult(true, false, result) + } + } catch (e: Exception) { + Logger.e(TAG, "Text input failed", e) + ActionResult(false, false, "文本输入失败: ${e.message}") + } + } + /** * Executes an agent action on the device. * @@ -180,7 +197,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.tap(absX, absY) + val result = deviceController.tap(absX, absY) if (isDeviceExecutorError(result)) { Logger.w(TAG, "Tap command failed: $result") ActionResult(false, false, "点击失败: $result") @@ -224,7 +241,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.swipe(swipePath.points, swipePath.durationMs) + val result = deviceController.swipe(swipePath.points, swipePath.durationMs) // Wait for swipe animation to complete before showing floating window // The swipe command returns immediately, but the gesture takes time @@ -259,9 +276,8 @@ class ActionHandler( return try { // Small delay to let the system settle focus delay(200) - - val result = textInputManager.typeText(action.text) - ActionResult(result.success, false, result.message) + + inputText(action.text) } finally { // Always show floating window after typing, even if typing fails showFloatingWindow() @@ -279,8 +295,8 @@ class ActionHandler( return try { // Small delay to let the system settle focus delay(200) - - val result = textInputManager.typeText(action.text) + + val result = inputText(action.text) ActionResult(result.success, false, "输入名称: ${action.text}") } finally { // Always show floating window after typing, even if typing fails @@ -310,7 +326,7 @@ class ActionHandler( return if (packageName != null) { Logger.i(TAG, "Launching package: $packageName") - val launchResult = deviceExecutor.launchApp(packageName) + val launchResult = deviceController.launchApp(packageName) // Check if launch was successful by examining the result val isError = launchResult.contains("Error", ignoreCase = true) || @@ -321,7 +337,7 @@ class ActionHandler( if (isError) { Logger.w(TAG, "Launch failed for $packageName: $launchResult") // Launch failed - instruct model to find app icon on screen - deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + deviceController.pressKey(IDeviceController.KEYCODE_HOME) ActionResult( success = true, // Operation itself succeeded, just app not found shouldFinish = false, @@ -334,7 +350,7 @@ class ActionHandler( // Package not found - instruct model to find app icon on screen Logger.i(TAG, "Package not found for '${action.app}', instructing model to find app icon on screen") // Press Home first to go to home screen - deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + deviceController.pressKey(IDeviceController.KEYCODE_HOME) ActionResult( success = true, shouldFinish = false, @@ -378,11 +394,11 @@ class ActionHandler( private suspend fun executeBack(): ActionResult { // First, dismiss keyboard with ESCAPE key to ensure Back actually navigates // If keyboard is shown, the first Back would just close it - deviceExecutor.pressKey(111) // KEYCODE_ESCAPE + deviceController.pressKey(111) // KEYCODE_ESCAPE delay(100) // Now press Back to navigate - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_BACK) + val result = deviceController.pressKey(IDeviceController.KEYCODE_BACK) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Back key press failed: $result") ActionResult(false, false, "返回键失败: $result") @@ -395,7 +411,7 @@ class ActionHandler( * Executes a Home action. */ private suspend fun executeHome(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + val result = deviceController.pressKey(IDeviceController.KEYCODE_HOME) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Home key press failed: $result") ActionResult(false, false, "主页键失败: $result") @@ -408,7 +424,7 @@ class ActionHandler( * Executes a VolumeUp action. */ private suspend fun executeVolumeUp(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_VOLUME_UP) + val result = deviceController.pressKey(IDeviceController.KEYCODE_VOLUME_UP) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Volume up key press failed: $result") ActionResult(false, false, "音量+键失败: $result") @@ -421,7 +437,7 @@ class ActionHandler( * Executes a VolumeDown action. */ private suspend fun executeVolumeDown(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_VOLUME_DOWN) + val result = deviceController.pressKey(IDeviceController.KEYCODE_VOLUME_DOWN) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Volume down key press failed: $result") ActionResult(false, false, "音量-键失败: $result") @@ -434,7 +450,7 @@ class ActionHandler( * Executes a Power action. */ private suspend fun executePower(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_POWER) + val result = deviceController.pressKey(IDeviceController.KEYCODE_POWER) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Power key press failed: $result") ActionResult(false, false, "电源键失败: $result") @@ -462,7 +478,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.longPress(absX, absY, action.durationMs) + val result = deviceController.longPress(absX, absY, action.durationMs) // Wait for long press to complete before showing floating window // The command returns immediately, but the gesture takes time @@ -498,7 +514,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.doubleTap(absX, absY) + val result = deviceController.doubleTap(absX, absY) if (isDeviceExecutorError(result)) { Logger.w(TAG, "Double tap command failed: $result") ActionResult(false, false, "双击失败: $result") diff --git a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt index e882e05..be82d36 100644 --- a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt +++ b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt @@ -197,15 +197,17 @@ class AccessibilityDeviceController( } if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { - Logger.w(TAG, "Screenshot not supported on Android < 13") + Logger.w(TAG, "Screenshot not supported on Android < 13 (API 33)") return@withContext createFallbackScreenshot() } try { + Logger.d(TAG, "Attempting to capture screenshot via AccessibilityService") val bitmap = service.captureScreen() if (bitmap != null) { val width = bitmap.width val height = bitmap.height + Logger.d(TAG, "Screenshot captured successfully: ${width}x${height}") // Convert bitmap to Screenshot val base64Data = encodeBitmapToBase64(bitmap) @@ -220,11 +222,14 @@ class AccessibilityDeviceController( isSensitive = false ) } else { - Logger.w(TAG, "Screenshot capture returned null") + Logger.w(TAG, "Screenshot capture returned null bitmap") createFallbackScreenshot() } + } catch (e: SecurityException) { + Logger.e(TAG, "Security exception when capturing screenshot - service may not have screenshot capability", e) + createFallbackScreenshot() } catch (e: Exception) { - Logger.e(TAG, "Failed to capture screenshot", e) + Logger.e(TAG, "Failed to capture screenshot via accessibility service", e) createFallbackScreenshot() } } diff --git a/app/src/main/res/xml/accessibility_service_config.xml b/app/src/main/res/xml/accessibility_service_config.xml index 37e7efd..8e4efdb 100644 --- a/app/src/main/res/xml/accessibility_service_config.xml +++ b/app/src/main/res/xml/accessibility_service_config.xml @@ -5,6 +5,7 @@ android:accessibilityFlags="flagDefault|flagRequestTouchExplorationMode|flagReportViewIds" android:canPerformGestures="true" android:canRetrieveWindowContent="true" + android:canTakeScreenshot="true" android:description="@string/accessibility_service_description" android:notificationTimeout="100" android:settingsActivity="com.kevinluo.autoglm.settings.SettingsActivity"> From 5e2b3bbf3eaf7420c4e51baadbe7fe3d4685c363 Mon Sep 17 00:00:00 2001 From: Repobor Date: Mon, 29 Dec 2025 22:06:16 +0800 Subject: [PATCH 5/6] feat: improve screenshot handling and permissions - Improve accessibility screenshot quality for compatibility with lower Android versions - Add notification permission support - Optimize Shizuku mode screenshot handling by directly transferring chunked screenshots instead of writing to files --- app/build.gradle.kts | 45 ++-- app/src/main/AndroidManifest.xml | 6 +- .../com/kevinluo/autoglm/IUserService.aidl | 6 + .../java/com/kevinluo/autoglm/MainActivity.kt | 120 +++++++++-- .../java/com/kevinluo/autoglm/UserService.kt | 59 ++++++ .../device/AccessibilityDeviceController.kt | 11 +- .../autoglm/history/HistoryDetailActivity.kt | 193 ++++++++++-------- .../autoglm/history/HistoryManager.kt | 120 ++++++----- .../autoglm/screenshot/ScreenshotService.kt | 182 +++++++---------- .../autoglm/ui/FloatingWindowTileService.kt | 10 +- app/src/main/res/values/strings.xml | 1 + .../autoglm/app/AppResolverPropertyTest.kt | 97 ++++----- 12 files changed, 508 insertions(+), 342 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index cf112f5..884bc98 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -10,10 +10,10 @@ android { defaultConfig { applicationId = "com.kevinluo.autoglm" - minSdk = 24 + minSdk = 27 targetSdk = 34 - versionCode = 5 - versionName = "0.0.5" + versionCode = 2 + versionName = "0.0.2" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } @@ -32,7 +32,8 @@ android { buildTypes { debug { - // 不再使用 applicationIdSuffix，与发行版使用相同包名 + applicationIdSuffix = ".debug" + versionNameSuffix = "-debug" resValue("string", "app_name", "AutoGLM Dev") } release { @@ -59,7 +60,7 @@ android { kotlinOptions { jvmTarget = "11" } - + // Enable JUnit 5 for Kotest property-based testing testOptions { unitTests.all { @@ -69,33 +70,25 @@ android { } } -// Copy dev_profiles.json to assets for debug builds only +// Copy dev_profiles.json to assets for debug builds android.applicationVariants.all { val variant = this - - // Custom APK file name - outputs.all { - val output = this as com.android.build.gradle.internal.api.BaseVariantOutputImpl - output.outputFileName = "AutoGLM-${variant.versionName}-${variant.buildType.name}.apk" - } - if (variant.buildType.name == "debug") { val copyDevProfiles = tasks.register("copyDevProfiles${variant.name.replaceFirstChar { it.uppercase() }}") { val devProfilesFile = rootProject.file("dev_profiles.json") - // Use debug-specific assets directory to avoid polluting release builds - val assetsDir = file("src/debug/assets") - + val assetsDir = file("src/main/assets") + doLast { if (devProfilesFile.exists()) { assetsDir.mkdirs() devProfilesFile.copyTo(File(assetsDir, "dev_profiles.json"), overwrite = true) - println("Copied dev_profiles.json to debug assets") + println("Copied dev_profiles.json to assets") } else { println("dev_profiles.json not found, skipping") } } } - + tasks.named("merge${variant.name.replaceFirstChar { it.uppercase() }}Assets") { dependsOn(copyDevProfiles) } @@ -111,29 +104,29 @@ dependencies { implementation(libs.androidx.constraintlayout) implementation(libs.shizuku.api) implementation(libs.shizuku.provider) - + // Kotlin Coroutines for async operations implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.coroutines.android) - + // Lifecycle & ViewModel implementation(libs.androidx.lifecycle.viewmodel) implementation(libs.androidx.lifecycle.runtime) - + // Security for encrypted preferences implementation(libs.androidx.security.crypto) - + // OkHttp for API communication implementation(libs.okhttp) implementation(libs.okhttp.sse) implementation(libs.okhttp.logging) - + // Retrofit for API communication implementation(libs.retrofit) - + // Kotlin Serialization for JSON parsing implementation(libs.kotlinx.serialization.json) - + // Testing testImplementation(libs.junit) testImplementation(libs.kotest.runner.junit5) @@ -141,4 +134,4 @@ dependencies { testImplementation(libs.kotest.assertions.core) androidTestImplementation(libs.androidx.junit) androidTestImplementation(libs.androidx.espresso.core) -} \ No newline at end of file +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index e9b7593..31ac67f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -3,9 +3,13 @@ xmlns:tools="http://schemas.android.com/tools"> + + + - \ No newline at end of file + diff --git a/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl b/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl index 09a179b..6ed15f5 100644 --- a/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl +++ b/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl @@ -3,4 +3,10 @@ package com.kevinluo.autoglm; interface IUserService { void destroy() = 16777114; String executeCommand(String command) = 1; + // Capture screenshot to internal buffer and return base64 length + int captureScreenshotAndGetSize() = 2; + // Read a base64 chunk from the internal screenshot buffer + String readScreenshotChunk(int offset, int size) = 3; + // Clear internal screenshot buffer to free memory + void clearScreenshotBuffer() = 4; } diff --git a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt index cb2d949..53c6f92 100644 --- a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt @@ -16,10 +16,11 @@ import android.widget.Button import android.widget.ImageButton import android.widget.TextView import android.widget.Toast -import androidx.activity.enableEdgeToEdge +import androidx.activity.result.contract.ActivityResultContracts import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity import androidx.core.content.ContextCompat +import androidx.core.content.PermissionChecker import androidx.core.view.ViewCompat import androidx.core.view.WindowInsetsCompat import androidx.lifecycle.lifecycleScope @@ -103,6 +104,21 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Component manager for dependency injection private lateinit var componentManager: ComponentManager + private val requestNotificationPermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted: Boolean -> + if (isGranted) { + Logger.i(TAG, "Notification permission granted") + } else { + Logger.w(TAG, "Notification permission denied") + Toast.makeText( + this, + R.string.toast_notification_permission_required, + Toast.LENGTH_LONG + ).show() + } + } + // Current step tracking for floating window private var currentStepNumber = 0 private var currentThinking = "" @@ -185,15 +201,8 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - enableEdgeToEdge() setContentView(R.layout.activity_main) - ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets -> - val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars()) - v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom) - insets - } - // Initialize ComponentManager componentManager = ComponentManager.getInstance(this) Logger.i(TAG, "ComponentManager initialized") @@ -203,9 +212,16 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { initViews() setupListeners() + checkAndRequestNotificationPermission() setupShizukuListeners() setupAccessibilityListener() + // For accessibility mode, initialize components immediately + val controlMode = componentManager.settingsManager.getDeviceControlMode() + if (controlMode == DeviceControlMode.ACCESSIBILITY) { + componentManager.reinitializeAgent() + } + updateDeviceServiceStatus() updateOverlayPermissionStatus() updateKeyboardStatus() @@ -338,6 +354,80 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { updateDeviceServiceStatus() } + /** + * Attempts to automatically enable AutoGLM Accessibility Service when in accessibility mode + * and the app has WRITE_SECURE_SETTINGS permission. + * + * This writes the component into Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES and sets + * Settings.Secure.ACCESSIBILITY_ENABLED to 1. Requires privileged permission. + */ + private fun tryAutoEnableAccessibilityService() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + if (controlMode != DeviceControlMode.ACCESSIBILITY) return + + // Already enabled + if (AutoGLMAccessibilityService.isEnabled()) return + + // Check WRITE_SECURE_SETTINGS permission + val hasWss = PermissionChecker.checkSelfPermission( + this, + android.Manifest.permission.WRITE_SECURE_SETTINGS + ) == PermissionChecker.PERMISSION_GRANTED + + if (!hasWss) { + Logger.d(TAG, "WRITE_SECURE_SETTINGS not granted; skip auto-enable") + return + } + + val component = ComponentName(this, AutoGLMAccessibilityService::class.java) + val flattened = component.flattenToString() + try { + val current = Settings.Secure.getString( + contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES + ) + val items = current?.split(":")?.filter { it.isNotBlank() }?.toMutableSet() ?: mutableSetOf() + if (!items.contains(flattened)) { + items.add(flattened) + } + val newValue = items.joinToString(":") + val updated = Settings.Secure.putString( + contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, + newValue + ) + val enabled = Settings.Secure.putInt( + contentResolver, + Settings.Secure.ACCESSIBILITY_ENABLED, + 1 + ) + Logger.i( + TAG, + "Auto-enable accessibility attempted: updated=$updated enabledSet=$enabled" + ) + } catch (e: Exception) { + Logger.e(TAG, "Failed to auto-enable accessibility service", e) + } + + // Refresh UI after attempt + updateAccessibilityStatus() + } + + /** + * Checks and requests notification permission on Android 13+. + */ + private fun checkAndRequestNotificationPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + if (ContextCompat.checkSelfPermission( + this, + android.Manifest.permission.POST_NOTIFICATIONS + ) != PackageManager.PERMISSION_GRANTED + ) { + requestNotificationPermissionLauncher.launch(android.Manifest.permission.POST_NOTIFICATIONS) + } + } + } + override fun onResume() { super.onResume() Logger.d(TAG, "onResume - checking for settings changes") @@ -351,6 +441,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Update accessibility status (user may have enabled it or mode changed) updateAccessibilityStatus() + // Attempt auto-enable accessibility if applicable + tryAutoEnableAccessibilityService() + // Re-setup floating window callbacks if service is running FloatingWindowService.getInstance()?.let { service -> service.setStopTaskCallback { @@ -374,12 +467,13 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } } - // Only reinitialize if service is connected and we need to refresh + // Check if settings actually changed before reinitializing + if (componentManager.settingsManager.hasConfigChanged()) { + componentManager.reinitializeAgent() + } + + // Update button states if (componentManager.isServiceConnected) { - // Check if settings actually changed before reinitializing - if (componentManager.settingsManager.hasConfigChanged()) { - componentManager.reinitializeAgent() - } componentManager.setPhoneAgentListener(this) setupConfirmationCallback() } diff --git a/app/src/main/java/com/kevinluo/autoglm/UserService.kt b/app/src/main/java/com/kevinluo/autoglm/UserService.kt index 909022f..9bac2ee 100644 --- a/app/src/main/java/com/kevinluo/autoglm/UserService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/UserService.kt @@ -9,6 +9,8 @@ import java.io.InputStreamReader * This service runs in a separate process with Shizuku permissions. */ class UserService : IUserService.Stub() { + // Internal buffer to store base64-encoded screenshot data + private var screenshotBase64: String? = null /** * Destroys the service and exits the process. @@ -57,6 +59,63 @@ class UserService : IUserService.Stub() { } } + /** + * Captures a screenshot and stores base64 data in memory. + * Returns the base64 string length, or -1 on failure. + */ + override fun captureScreenshotAndGetSize(): Int { + return try { + val process = Runtime.getRuntime().exec(arrayOf("sh", "-c", "screencap -p | base64")) + val reader = BufferedReader(InputStreamReader(process.inputStream)) + val errorReader = BufferedReader(InputStreamReader(process.errorStream)) + + val sb = StringBuilder() + var line: String? + // Read without preserving newlines to avoid wrapping issues + while (reader.readLine().also { line = it } != null) { + sb.append(line) + } + + val errorOutput = StringBuilder() + while (errorReader.readLine().also { line = it } != null) { + errorOutput.append(line).append('\n') + } + + val exitCode = process.waitFor() + reader.close() + errorReader.close() + + if (exitCode != 0 || errorOutput.isNotEmpty()) { + // Failure, clear buffer + screenshotBase64 = null + return -1 + } + + screenshotBase64 = sb.toString() + screenshotBase64?.length ?: -1 + } catch (e: Exception) { + screenshotBase64 = null + -1 + } + } + + /** + * Reads a chunk from the in-memory base64 screenshot. + */ + override fun readScreenshotChunk(offset: Int, size: Int): String { + val data = screenshotBase64 ?: return "" + if (offset < 0 || size <= 0 || offset >= data.length) return "" + val end = kotlin.math.min(offset + size, data.length) + return data.substring(offset, end) + } + + /** + * Clears the in-memory screenshot buffer. + */ + override fun clearScreenshotBuffer() { + screenshotBase64 = null + } + companion object { private const val TAG = "UserService" } diff --git a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt index be82d36..c65fb59 100644 --- a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt +++ b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt @@ -262,11 +262,18 @@ class AccessibilityDeviceController( */ private fun encodeBitmapToBase64( bitmap: Bitmap, - format: Bitmap.CompressFormat = Bitmap.CompressFormat.WEBP_LOSSY, + format: Bitmap.CompressFormat? = null, quality: Int = 80 ): String { + val compressFormat = format ?: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + val outputStream = ByteArrayOutputStream() - bitmap.compress(format, quality, outputStream) + bitmap.compress(compressFormat, quality, outputStream) return Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) } diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt index 3bba2d7..7c5e41e 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt @@ -50,28 +50,28 @@ import java.util.Locale * */ class HistoryDetailActivity : AppCompatActivity() { - + private lateinit var historyManager: HistoryManager private var taskId: String? = null private var task: TaskHistory? = null - + private lateinit var contentRecyclerView: RecyclerView private lateinit var detailAdapter: HistoryDetailAdapter - + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()) - + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_history_detail) - + historyManager = HistoryManager.getInstance(this) taskId = intent.getStringExtra(EXTRA_TASK_ID) - + Logger.d(TAG, "HistoryDetailActivity created for task: $taskId") setupViews() loadTask() } - + /** * Sets up all view references and click listeners. */ @@ -79,23 +79,23 @@ class HistoryDetailActivity : AppCompatActivity() { findViewById(R.id.backBtn).setOnClickListener { finish() } - + findViewById(R.id.copyPromptBtn).setOnClickListener { copyPromptToClipboard() } - + findViewById(R.id.saveImageBtn).setOnClickListener { saveAsImage() } - + findViewById(R.id.shareBtn).setOnClickListener { shareAsImage() } - + findViewById(R.id.deleteBtn).setOnClickListener { showDeleteDialog() } - + // Setup RecyclerView with true recycling contentRecyclerView = findViewById(R.id.contentRecyclerView) detailAdapter = HistoryDetailAdapter(historyManager, lifecycleScope) @@ -107,27 +107,27 @@ class HistoryDetailActivity : AppCompatActivity() { setItemViewCacheSize(3) } } - + /** * Loads the task from history manager. */ private fun loadTask() { val id = taskId ?: return - + lifecycleScope.launch { task = historyManager.getTask(id) - task?.let { + task?.let { Logger.d(TAG, "Loaded task with ${it.stepCount} steps") - detailAdapter.setTask(it) + detailAdapter.setTask(it) } } } - + override fun onDestroy() { super.onDestroy() detailAdapter.cleanup() } - + /** * Shows a confirmation dialog for deleting the task. */ @@ -140,22 +140,22 @@ class HistoryDetailActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + /** * Copies the task prompt/description to clipboard. * */ private fun copyPromptToClipboard() { val currentTask = task ?: return - + val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager val clip = ClipData.newPlainText("AutoGLM Prompt", currentTask.taskDescription) clipboard.setPrimaryClip(clip) - + Logger.d(TAG, "Copied prompt to clipboard") Toast.makeText(this, R.string.history_prompt_copied, Toast.LENGTH_SHORT).show() } - + /** * Deletes the current task from history. */ @@ -167,7 +167,7 @@ class HistoryDetailActivity : AppCompatActivity() { finish() } } - + /** * Formats duration in milliseconds to a human-readable string. * @@ -182,7 +182,7 @@ class HistoryDetailActivity : AppCompatActivity() { else -> "${seconds / 3600}时${(seconds % 3600) / 60}分" } } - + /** * Saves the task history as an image to gallery. * @@ -191,22 +191,22 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun saveAsImage() { val currentTask = task ?: return - + Logger.d(TAG, "Saving task as image") Toast.makeText(this, R.string.history_generating_image, Toast.LENGTH_SHORT).show() - + lifecycleScope.launch { try { val bitmap = withContext(Dispatchers.Default) { generateShareImage(currentTask) } - + val saved = withContext(Dispatchers.IO) { saveBitmapToGallery(bitmap) } - + bitmap.recycle() - + if (saved) { Logger.d(TAG, "Image saved to gallery") Toast.makeText(this@HistoryDetailActivity, R.string.history_save_success, Toast.LENGTH_SHORT).show() @@ -220,7 +220,7 @@ class HistoryDetailActivity : AppCompatActivity() { } } } - + /** * Shares the task history as an image. * @@ -229,24 +229,24 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun shareAsImage() { val currentTask = task ?: return - + Logger.d(TAG, "Sharing task as image") Toast.makeText(this, R.string.history_generating_image, Toast.LENGTH_SHORT).show() - + lifecycleScope.launch { try { val bitmap = withContext(Dispatchers.Default) { generateShareImage(currentTask) } - + // Save bitmap to cache directory val file = withContext(Dispatchers.IO) { saveBitmapToCache(bitmap) } - + // Share the image shareImageFile(file) - + // Recycle bitmap bitmap.recycle() } catch (e: Exception) { @@ -255,7 +255,7 @@ class HistoryDetailActivity : AppCompatActivity() { } } } - + /** * Generates a share image from task history. * @@ -270,16 +270,16 @@ class HistoryDetailActivity : AppCompatActivity() { val contentWidth = width - padding * 2 val stepSpacing = 40 // Spacing between steps val screenshotWidthRatio = 0.8f // Screenshot width = 80% of image width - + // Calculate heights val headerHeight = 200 val stepBaseHeight = 120 val thinkingLineHeight = 50 - + // First pass: calculate total height (need to load screenshots to get their heights) var totalHeight = headerHeight + padding * 2 val screenshotHeights = mutableMapOf() - + for ((index, step) in task.steps.withIndex()) { totalHeight += stepBaseHeight if (step.thinking.isNotBlank()) { @@ -301,52 +301,52 @@ class HistoryDetailActivity : AppCompatActivity() { totalHeight += stepSpacing } totalHeight += 80 // footer - + // Create bitmap val bitmap = Bitmap.createBitmap(width, totalHeight, Bitmap.Config.ARGB_8888) val canvas = Canvas(bitmap) - + // Background canvas.drawColor(Color.parseColor("#1A1A1A")) - + // Paints val titlePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.WHITE textSize = 48f typeface = Typeface.DEFAULT_BOLD } - + val subtitlePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#AAAAAA") textSize = 32f } - + val stepNumberPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#4CAF50") textSize = 36f typeface = Typeface.DEFAULT_BOLD } - + val actionPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.WHITE textSize = 34f } - + val thinkingPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#888888") textSize = 28f } - + val cardPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#2A2A2A") } - + var y = padding.toFloat() - + // Draw header canvas.drawText("AutoGLM 任务记录", padding.toFloat(), y + 50, titlePaint) y += 70 - + // Task description val descLines = wrapText(task.taskDescription, actionPaint, contentWidth.toFloat()) for (line in descLines) { @@ -354,18 +354,18 @@ class HistoryDetailActivity : AppCompatActivity() { y += 45 } y += 20 - + // Status and info val statusColor = if (task.success) Color.parseColor("#4CAF50") else Color.parseColor("#F44336") val statusPaint = Paint(subtitlePaint).apply { color = statusColor } val statusText = if (task.success) "✓ 成功" else "✗ 失败" canvas.drawText(statusText, padding.toFloat(), y + 35, statusPaint) - + val duration = formatDuration(task.duration) val infoStr = "${dateFormat.format(Date(task.startTime))} · ${task.stepCount}步 · $duration" canvas.drawText(infoStr, padding + 150f, y + 35, subtitlePaint) y += 60 - + // Draw steps for ((index, step) in task.steps.withIndex()) { // Add spacing before each step (except first one uses smaller spacing) @@ -374,7 +374,7 @@ class HistoryDetailActivity : AppCompatActivity() { } else { y += stepSpacing } - + // Step card background val cardTop = y var cardHeight = stepBaseHeight.toFloat() @@ -385,32 +385,32 @@ class HistoryDetailActivity : AppCompatActivity() { screenshotHeights[index]?.let { h -> cardHeight += h + 40 } - + canvas.drawRoundRect( padding.toFloat(), cardTop, (width - padding).toFloat(), cardTop + cardHeight, 20f, 20f, cardPaint ) - + y += 15 - + // Step number canvas.drawText("步骤 ${step.stepNumber}", padding + 20f, y + 40, stepNumberPaint) - + // Status indicator val stepStatusPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = if (step.success) Color.parseColor("#4CAF50") else Color.parseColor("#F44336") } canvas.drawCircle(width - padding - 30f, y + 25, 10f, stepStatusPaint) y += 50 - + // Action description val actionLines = wrapText(step.actionDescription, actionPaint, contentWidth - 40f) for (line in actionLines) { canvas.drawText(line, padding + 20f, y + 30, actionPaint) y += 40 } - + // Thinking if (step.thinking.isNotBlank()) { y += 10 @@ -420,7 +420,7 @@ class HistoryDetailActivity : AppCompatActivity() { y += 35 } } - + // Screenshot val screenshotPath = step.annotatedScreenshotPath ?: step.screenshotPath if (screenshotPath != null) { @@ -432,29 +432,29 @@ class HistoryDetailActivity : AppCompatActivity() { val scale = targetWidth.toFloat() / screenshotBitmap.width val scaledWidth = targetWidth val scaledHeight = (screenshotBitmap.height * scale).toInt() - + val scaledBitmap = Bitmap.createScaledBitmap(screenshotBitmap, scaledWidth, scaledHeight, true) val left = (width - scaledWidth) / 2 // Center horizontally canvas.drawBitmap(scaledBitmap, left.toFloat(), y, null) - + y += scaledHeight scaledBitmap.recycle() screenshotBitmap.recycle() } } - + // Move y to end of card (cardTop + cardHeight) y = cardTop + cardHeight } - + // Footer y += 30 val footerPaint = Paint(subtitlePaint).apply { textSize = 24f } canvas.drawText("由 AutoGLM For Android 生成", padding.toFloat(), y + 30, footerPaint) - + return bitmap } - + /** * Wraps text to fit within a given width. * @@ -466,11 +466,11 @@ class HistoryDetailActivity : AppCompatActivity() { private fun wrapText(text: String, paint: Paint, maxWidth: Float): List { val lines = mutableListOf() var remaining = text - + while (remaining.isNotEmpty()) { val count = paint.breakText(remaining, true, maxWidth, null) if (count == 0) break - + // Try to break at word boundary var breakAt = count if (count < remaining.length) { @@ -479,14 +479,14 @@ class HistoryDetailActivity : AppCompatActivity() { breakAt = lastSpace + 1 } } - + lines.add(remaining.substring(0, breakAt).trim()) remaining = remaining.substring(breakAt).trim() } - + return lines.ifEmpty { listOf("") } } - + /** * Saves bitmap to cache directory. * @@ -496,14 +496,21 @@ class HistoryDetailActivity : AppCompatActivity() { private fun saveBitmapToCache(bitmap: Bitmap): File { val cacheDir = File(cacheDir, "share") if (!cacheDir.exists()) cacheDir.mkdirs() - + + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + val file = File(cacheDir, "autoglm_task_${System.currentTimeMillis()}.webp") FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + bitmap.compress(compressFormat, 90, out) } return file } - + /** * Shares the image file using system share sheet. * @@ -515,16 +522,16 @@ class HistoryDetailActivity : AppCompatActivity() { "${packageName}.fileprovider", file ) - + val intent = Intent(Intent.ACTION_SEND).apply { type = "image/webp" putExtra(Intent.EXTRA_STREAM, uri) addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) } - + startActivity(Intent.createChooser(intent, getString(R.string.history_share_title))) } - + /** * Saves bitmap to device gallery. * @@ -535,7 +542,7 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun saveBitmapToGallery(bitmap: Bitmap): Boolean { val filename = "AutoGLM_${System.currentTimeMillis()}.webp" - + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { // Android 10+ use MediaStore val contentValues = ContentValues().apply { @@ -544,15 +551,21 @@ class HistoryDetailActivity : AppCompatActivity() { put(MediaStore.Images.Media.RELATIVE_PATH, Environment.DIRECTORY_PICTURES + "/AutoGLM") put(MediaStore.Images.Media.IS_PENDING, 1) } - + val resolver = contentResolver val uri = resolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues) - + uri?.let { resolver.openOutputStream(it)?.use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, 90, out) } - + contentValues.clear() contentValues.put(MediaStore.Images.Media.IS_PENDING, 0) resolver.update(it, contentValues, null, null) @@ -564,10 +577,16 @@ class HistoryDetailActivity : AppCompatActivity() { val picturesDir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES) val autoglmDir = File(picturesDir, "AutoGLM") if (!autoglmDir.exists()) autoglmDir.mkdirs() - + val file = File(autoglmDir, filename) FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, 90, out) } // Notify gallery using MediaScannerConnection (recommended approach) @@ -582,10 +601,10 @@ class HistoryDetailActivity : AppCompatActivity() { true } } - + companion object { private const val TAG = "HistoryDetailActivity" - + /** Intent extra key for task ID. */ const val EXTRA_TASK_ID = "task_id" } diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt index 0ca4298..101bb53 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt @@ -3,6 +3,7 @@ package com.kevinluo.autoglm.history import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.os.Build import android.util.Base64 import com.kevinluo.autoglm.action.AgentAction import com.kevinluo.autoglm.util.Logger @@ -38,33 +39,33 @@ import java.util.Locale * */ class HistoryManager private constructor(private val context: Context) { - + /** Directory for storing task history files. */ private val historyDir: File by lazy { File(context.filesDir, HISTORY_DIR).also { it.mkdirs() } } - + /** Currently recording task, null if no task is being recorded. */ private var currentTask: TaskHistory? = null - + /** Base64-encoded screenshot data for the current step. */ private var currentScreenshotBase64: String? = null - + /** Width of the current screenshot in pixels. */ private var currentScreenshotWidth: Int = 0 - + /** Height of the current screenshot in pixels. */ private var currentScreenshotHeight: Int = 0 - + private val _historyList = MutableStateFlow>(emptyList()) - + /** Observable list of all task histories, sorted by most recent first. */ val historyList: StateFlow> = _historyList.asStateFlow() - + init { loadHistoryIndex() } - + /** * Starts recording a new task. * @@ -81,7 +82,7 @@ class HistoryManager private constructor(private val context: Context) { Logger.d(TAG, "Started recording task: ${task.id}") return task } - + /** * Sets the current screenshot for the next step. * @@ -123,19 +124,19 @@ class HistoryManager private constructor(private val context: Context) { message: String? = null ) = withContext(Dispatchers.IO) { val task = currentTask ?: return@withContext - + var screenshotPath: String? = null var annotatedPath: String? = null - + // Save screenshot if available currentScreenshotBase64?.let { base64 -> try { // Decode base64 to raw bytes (already WebP format) val webpBytes = Base64.decode(base64, Base64.DEFAULT) - + // Save original screenshot directly without re-compression screenshotPath = saveScreenshotBytes(task.id, stepNumber, webpBytes, false) - + // Create and save annotated screenshot if action has visual annotation if (action != null) { val annotation = ScreenshotAnnotator.createAnnotation( @@ -163,7 +164,7 @@ class HistoryManager private constructor(private val context: Context) { Logger.e(TAG, "Failed to save screenshot for step $stepNumber", e) } } - + val step = HistoryStep( stepNumber = stepNumber, thinking = thinking, @@ -174,14 +175,14 @@ class HistoryManager private constructor(private val context: Context) { success = success, message = message ) - + task.steps.add(step) Logger.d(TAG, "Recorded step $stepNumber for task ${task.id}") - + // Clear current screenshot currentScreenshotBase64 = null } - + /** * Completes the current task recording. * @@ -195,38 +196,38 @@ class HistoryManager private constructor(private val context: Context) { */ suspend fun completeTask(success: Boolean, message: String?) = withContext(Dispatchers.IO) { val task = currentTask ?: return@withContext - + // Don't save empty tasks (no steps recorded) if (task.steps.isEmpty()) { Logger.d(TAG, "Skipping empty task ${task.id}") currentTask = null return@withContext } - + task.endTime = System.currentTimeMillis() task.success = success task.completionMessage = message - + // Save task to disk saveTask(task) - + // Update history list val updatedList = _historyList.value.toMutableList() updatedList.add(0, task) - + // Trim old history if needed while (updatedList.size > MAX_HISTORY_COUNT) { val removed = updatedList.removeAt(updatedList.size - 1) deleteTaskFiles(removed.id) } - + _historyList.value = updatedList saveHistoryIndex() - + Logger.d(TAG, "Completed task ${task.id}, success=$success") currentTask = null } - + /** * Gets a task history by ID. * @@ -239,7 +240,7 @@ class HistoryManager private constructor(private val context: Context) { suspend fun getTask(taskId: String): TaskHistory? = withContext(Dispatchers.IO) { loadTask(taskId) } - + /** * Deletes a task history. * @@ -253,7 +254,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = _historyList.value.filter { it.id != taskId } saveHistoryIndex() } - + /** * Deletes multiple task histories. * @@ -269,7 +270,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = _historyList.value.filter { it.id !in taskIds } saveHistoryIndex() } - + /** * Clears all history. * @@ -281,7 +282,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = emptyList() saveHistoryIndex() } - + /** * Gets the screenshot bitmap for a step. * @@ -297,9 +298,9 @@ class HistoryManager private constructor(private val context: Context) { if (!file.exists()) return null return BitmapFactory.decodeFile(path) } - + // Private helper methods - + /** * Saves raw WebP bytes directly to file (no re-compression). * @@ -318,14 +319,14 @@ class HistoryManager private constructor(private val context: Context) { val taskDir = File(historyDir, taskId).also { it.mkdirs() } val suffix = if (annotated) "_annotated" else "" val file = File(taskDir, "step_${stepNumber}${suffix}.webp") - + FileOutputStream(file).use { out -> out.write(webpBytes) } - + return file.absolutePath } - + /** * Saves bitmap as WebP (used for annotated screenshots). * @@ -344,14 +345,21 @@ class HistoryManager private constructor(private val context: Context) { val taskDir = File(historyDir, taskId).also { it.mkdirs() } val suffix = if (annotated) "_annotated" else "" val file = File(taskDir, "step_${stepNumber}${suffix}.webp") - + + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 85, out) + bitmap.compress(compressFormat, 85, out) } - + return file.absolutePath } - + /** * Decodes a Base64 string to a Bitmap. * @@ -367,7 +375,7 @@ class HistoryManager private constructor(private val context: Context) { null } } - + /** * Saves a task's metadata to JSON file. * @@ -376,7 +384,7 @@ class HistoryManager private constructor(private val context: Context) { private fun saveTask(task: TaskHistory) { val taskDir = File(historyDir, task.id).also { it.mkdirs() } val metaFile = File(taskDir, "meta.json") - + val json = JSONObject().apply { put("id", task.id) put("taskDescription", task.taskDescription) @@ -384,7 +392,7 @@ class HistoryManager private constructor(private val context: Context) { put("endTime", task.endTime) put("success", task.success) put("completionMessage", task.completionMessage) - + val stepsArray = JSONArray() task.steps.forEach { step -> stepsArray.put(JSONObject().apply { @@ -400,10 +408,10 @@ class HistoryManager private constructor(private val context: Context) { } put("steps", stepsArray) } - + metaFile.writeText(json.toString(2)) } - + /** * Loads a task from its JSON metadata file. * @@ -413,11 +421,11 @@ class HistoryManager private constructor(private val context: Context) { private fun loadTask(taskId: String): TaskHistory? { val metaFile = File(historyDir, "$taskId/meta.json") if (!metaFile.exists()) return null - + return try { val json = JSONObject(metaFile.readText()) val steps = mutableListOf() - + val stepsArray = json.optJSONArray("steps") if (stepsArray != null) { for (i in 0 until stepsArray.length()) { @@ -436,7 +444,7 @@ class HistoryManager private constructor(private val context: Context) { )) } } - + TaskHistory( id = json.getString("id"), taskDescription = json.getString("taskDescription"), @@ -451,7 +459,7 @@ class HistoryManager private constructor(private val context: Context) { null } } - + /** * Deletes all files associated with a task. * @@ -460,7 +468,7 @@ class HistoryManager private constructor(private val context: Context) { private fun deleteTaskFiles(taskId: String) { File(historyDir, taskId).deleteRecursively() } - + /** * Loads the history index from persistent storage. * @@ -469,22 +477,22 @@ class HistoryManager private constructor(private val context: Context) { private fun loadHistoryIndex() { val indexFile = File(historyDir, INDEX_FILE) if (!indexFile.exists()) return - + try { val json = JSONArray(indexFile.readText()) val list = mutableListOf() - + for (i in 0 until json.length()) { val taskId = json.getString(i) loadTask(taskId)?.let { list.add(it) } } - + _historyList.value = list } catch (e: Exception) { Logger.e(TAG, "Failed to load history index", e) } } - + /** * Saves the history index to persistent storage. * @@ -496,16 +504,16 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value.forEach { json.put(it.id) } indexFile.writeText(json.toString()) } - + companion object { private const val TAG = "HistoryManager" private const val HISTORY_DIR = "task_history" private const val INDEX_FILE = "history_index.json" private const val MAX_HISTORY_COUNT = 50 - + @Volatile private var instance: HistoryManager? = null - + /** * Gets the singleton instance of HistoryManager. * diff --git a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt index ffd6fc2..17b9285 100644 --- a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt @@ -2,6 +2,7 @@ package com.kevinluo.autoglm.screenshot import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.os.Build import android.util.Base64 import com.kevinluo.autoglm.IUserService import com.kevinluo.autoglm.util.ErrorHandler @@ -98,19 +99,19 @@ class ScreenshotService( private const val SHOW_DELAY_MS = 100L private const val FALLBACK_WIDTH = 1080 private const val FALLBACK_HEIGHT = 1920 - + // Screenshot compression settings - optimized for API upload private const val WEBP_QUALITY = 65 // Reduced from 70 for better compression - + // Screenshot scaling settings - use max dimensions instead of fixed scale factor // This ensures consistent output size regardless of device resolution private const val MAX_WIDTH = 720 // Max width after scaling private const val MAX_HEIGHT = 1280 // Max height after scaling - + // Base64 output chunk size for reading (safe for Binder) private const val BASE64_CHUNK_SIZE = 500000 } - + /** * Captures the current screen content. * @@ -125,7 +126,7 @@ class ScreenshotService( val floatingWindowController = floatingWindowControllerProvider() val hasFloatingWindow = floatingWindowController != null Logger.d(TAG, "Starting screenshot capture, window visible: ${floatingWindowController?.isVisible()}") - + // Hide floating window before capture if (hasFloatingWindow) { Logger.d(TAG, "Hiding floating window") @@ -134,7 +135,7 @@ class ScreenshotService( } delay(HIDE_DELAY_MS) } - + try { // Capture screenshot val result = captureScreen() @@ -155,7 +156,7 @@ class ScreenshotService( } } } - + /** * Captures the screen using Shizuku shell command. * @@ -213,16 +214,22 @@ class ScreenshotService( // Convert to WebP for better compression val webpStream = ByteArrayOutputStream() - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, WEBP_QUALITY, webpStream) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, WEBP_QUALITY, webpStream) bitmap.recycle() - + val webpData = webpStream.toByteArray() val compressionRatio = if (pngData.isNotEmpty()) 100 * webpData.size / pngData.size else 0 Logger.d(TAG, "Converted to WebP: ${webpData.size} bytes ($compressionRatio% of PNG)") - + val base64Data = encodeToBase64(webpData) Logger.d(TAG, "Screenshot captured: ${scaledWidth}x${scaledHeight}, base64 length: ${base64Data.length}") - + Screenshot( base64Data = base64Data, width = scaledWidth, @@ -236,7 +243,7 @@ class ScreenshotService( createFallbackScreenshot() } } - + /** * Calculates optimal dimensions based on max size constraints. * @@ -254,21 +261,21 @@ class ScreenshotService( Logger.d(TAG, "Image already within limits: ${originalWidth}x${originalHeight}") return originalWidth to originalHeight } - + // Calculate scale ratios for both dimensions val widthRatio = MAX_WIDTH.toFloat() / originalWidth val heightRatio = MAX_HEIGHT.toFloat() / originalHeight - + // Use the smaller ratio to ensure both dimensions fit within limits val ratio = minOf(widthRatio, heightRatio) - + val scaledWidth = (originalWidth * ratio).toInt() val scaledHeight = (originalHeight * ratio).toInt() - + Logger.d(TAG, "Scaling with ratio $ratio: ${originalWidth}x${originalHeight} -> ${scaledWidth}x${scaledHeight}") return scaledWidth to scaledHeight } - + /** * Executes screencap command and returns raw bytes. * @@ -284,106 +291,59 @@ class ScreenshotService( return@coroutineScope null } - val timestamp = System.currentTimeMillis() - val pngFile = "/data/local/tmp/screenshot_$timestamp.png" - val base64File = "$pngFile.b64" - try { - Logger.d(TAG, "Attempting screenshot capture") + Logger.d(TAG, "Attempting screenshot capture (streamed, no temp files)") val startTime = System.currentTimeMillis() - // Capture screenshot and pipe to base64 - val captureResult = service.executeCommand( - "screencap -p | base64 > $base64File && stat -c %s $base64File" - ) - - val captureTime = System.currentTimeMillis() - startTime - Logger.d(TAG, "Screenshot capture took ${captureTime}ms") - - // Check for errors - if (captureResult.contains("Error") || captureResult.contains("permission denied", ignoreCase = true)) { - Logger.w(TAG, "Screenshot capture failed: $captureResult") + // Trigger capture in the service and get total base64 size + val totalSize = service.captureScreenshotAndGetSize() + if (totalSize <= 0) { + Logger.w(TAG, "Screenshot capture failed or returned empty data") return@coroutineScope null } - - // Parse base64 file size from output - val base64Size = captureResult.lines() - .firstOrNull { it.trim().all { c -> c.isDigit() } } - ?.trim()?.toLongOrNull() ?: 0L - - if (base64Size == 0L) { - Logger.w(TAG, "Base64 file not created or empty") - return@coroutineScope null - } - - Logger.d(TAG, "Base64 file size: $base64Size bytes") - - // Read base64 file + val readStartTime = System.currentTimeMillis() - val base64Data: String - - if (base64Size <= BASE64_CHUNK_SIZE) { - // Small file - read in one go - val result = userService.executeCommand("cat $base64File") - base64Data = result.lines() - .filter { line -> - !line.startsWith("[") && - line.isNotBlank() && - !line.contains("exit code", ignoreCase = true) - } - .joinToString("") - } else { - // Large file - read chunks sequentially to avoid Binder buffer overflow - val chunkCount = ((base64Size + BASE64_CHUNK_SIZE - 1) / BASE64_CHUNK_SIZE).toInt() - Logger.d(TAG, "Reading $chunkCount chunks sequentially") - - val chunks = mutableListOf() - - for (index in 0 until chunkCount) { - val offset = index.toLong() * BASE64_CHUNK_SIZE - val remaining = base64Size - offset - val currentChunkSize = minOf(BASE64_CHUNK_SIZE.toLong(), remaining) - - val chunkResult = userService.executeCommand( - "tail -c +${offset + 1} $base64File | head -c $currentChunkSize" - ) - - val chunkData = chunkResult.lines() - .filter { line -> - !line.startsWith("[") && - line.isNotBlank() && - !line.contains("exit code", ignoreCase = true) - } - .joinToString("") - - chunks.add(chunkData) + val chunkCount = ((totalSize + BASE64_CHUNK_SIZE - 1) / BASE64_CHUNK_SIZE) + Logger.d(TAG, "Reading $chunkCount chunks sequentially, total size: $totalSize") + + val chunks = StringBuilder(totalSize) + var offset = 0 + for (i in 0 until chunkCount) { + val currentChunkSize = kotlin.math.min(BASE64_CHUNK_SIZE, totalSize - offset) + val chunk = service.readScreenshotChunk(offset, currentChunkSize) + if (chunk.isEmpty()) { + Logger.w(TAG, "Empty chunk at index $i, offset=$offset size=$currentChunkSize") + // Bail out and return null to fallback + return@coroutineScope null } - - base64Data = chunks.joinToString("") + chunks.append(chunk) + offset += currentChunkSize } - + + val base64Data = chunks.toString() val readTime = System.currentTimeMillis() - readStartTime - Logger.d(TAG, "Base64 read took ${readTime}ms, total length: ${base64Data.length}") - + val captureTime = System.currentTimeMillis() - startTime + Logger.d(TAG, "Base64 read took ${readTime}ms, capture total ${captureTime}ms, length=${base64Data.length}") + if (base64Data.isBlank()) { Logger.w(TAG, "No base64 data read") return@coroutineScope null } - + Base64.decode(base64Data, Base64.DEFAULT) } catch (e: Exception) { - Logger.e(TAG, "Failed to capture screenshot to bytes", e) + Logger.e(TAG, "Failed to capture screenshot to bytes (stream)", e) null } finally { - // Clean up temp files + // Always clear internal buffer after reading try { - userService.executeCommand("rm -f $pngFile $base64File") + service.clearScreenshotBuffer() } catch (_: Exception) { // Ignore cleanup errors } } } - + /** * Creates a fallback black screenshot in WebP format. * @@ -395,13 +355,19 @@ class ScreenshotService( */ private fun createFallbackScreenshot(): Screenshot { val bitmap = Bitmap.createBitmap(FALLBACK_WIDTH, FALLBACK_HEIGHT, Bitmap.Config.ARGB_8888) - + val outputStream = ByteArrayOutputStream() - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, WEBP_QUALITY, outputStream) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, WEBP_QUALITY, outputStream) bitmap.recycle() - + val base64Data = Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) - + return Screenshot( base64Data = base64Data, width = FALLBACK_WIDTH, @@ -409,7 +375,7 @@ class ScreenshotService( isSensitive = true ) } - + /** * Encodes byte array to base64 string. * @@ -420,7 +386,7 @@ class ScreenshotService( fun encodeToBase64(data: ByteArray): String { return Base64.encodeToString(data, Base64.NO_WRAP) } - + /** * Decodes base64 string to byte array. * @@ -431,7 +397,7 @@ class ScreenshotService( fun decodeFromBase64(base64Data: String): ByteArray { return Base64.decode(base64Data, Base64.DEFAULT) } - + /** * Decodes base64 screenshot data to a Bitmap. * @@ -447,7 +413,7 @@ class ScreenshotService( null } } - + /** * Encodes a Bitmap to base64 string. * @@ -459,14 +425,20 @@ class ScreenshotService( */ fun encodeBitmapToBase64( bitmap: Bitmap, - format: Bitmap.CompressFormat = Bitmap.CompressFormat.WEBP_LOSSY, + format: Bitmap.CompressFormat? = null, quality: Int = WEBP_QUALITY ): String { + val compressFormat = format ?: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } val outputStream = ByteArrayOutputStream() - bitmap.compress(format, quality, outputStream) + bitmap.compress(compressFormat, quality, outputStream) return encodeToBase64(outputStream.toByteArray()) } - + /** * Creates a Screenshot object from a Bitmap. * diff --git a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt index 8cf79ec..1521690 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt @@ -47,7 +47,7 @@ class FloatingWindowTileService : TileService() { } startActivityAndCollapseCompat(intent) } - + /** * Compatibility wrapper for startActivityAndCollapse. * API 34+ requires PendingIntent, older versions use Intent directly. @@ -64,7 +64,7 @@ class FloatingWindowTileService : TileService() { startActivityAndCollapse(pendingIntent) } else { // API < 34 - @Suppress("DEPRECATION") + @Suppress("StartActivityAndCollapseDeprecated") startActivityAndCollapse(intent) } } @@ -88,14 +88,14 @@ class FloatingWindowTileService : TileService() { private fun updateTileState() { val tile = qsTile ?: return - + val service = FloatingWindowService.getInstance() val isVisible = service?.isVisible() == true - + tile.state = if (isVisible) Tile.STATE_ACTIVE else Tile.STATE_INACTIVE tile.label = getString(com.kevinluo.autoglm.R.string.tile_floating_window) tile.contentDescription = getString(com.kevinluo.autoglm.R.string.tile_floating_window_desc) - + tile.updateTile() } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index ca0e062..afd5413 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -61,6 +61,7 @@ 任务已开始 任务已取消 需要悬浮窗权限 + 后台服务需要通知权限 设置 diff --git a/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt b/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt index cb1e11a..f55e6b0 100644 --- a/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt +++ b/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt @@ -7,42 +7,42 @@ import io.kotest.property.forAll /** * Property-based tests for AppResolver. - * + * * **Feature: autoglm-phone-agent, Property 20: App name resolution consistency** * **Validates: Requirements 9.1** - * + * * These tests verify the core similarity calculation and matching logic * of the AppResolver without requiring Android framework dependencies. */ class AppResolverPropertyTest : StringSpec({ - + /** * Property 20: App name resolution consistency - * + * * For any installed app with a known display name, querying the AppResolver * with that exact name should return the correct package name. - * + * * Since we can't use PackageManager in unit tests, we test the underlying * similarity calculation logic which is the core of the resolution algorithm. - * + * * **Validates: Requirements 9.1** */ - + // Test that exact matches always return similarity of 1.0 "exact match should always return similarity of 1.0" { val stringArb = Arb.string(1..50, Codepoint.alphanumeric()) - + forAll(100, stringArb) { name -> val resolver = createTestableAppResolver() val similarity = resolver.calculateSimilarity(name.lowercase(), name.lowercase()) similarity == 1.0 } } - + // Test that similarity is symmetric for Levenshtein distance "levenshtein distance should be symmetric" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { s1, s2 -> val resolver = createTestableAppResolver() val dist1 = resolver.levenshteinDistance(s1, s2) @@ -50,33 +50,33 @@ class AppResolverPropertyTest : StringSpec({ dist1 == dist2 } } - + // Test that levenshtein distance is non-negative "levenshtein distance should be non-negative" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { s1, s2 -> val resolver = createTestableAppResolver() val distance = resolver.levenshteinDistance(s1, s2) distance >= 0 } } - + // Test that levenshtein distance is zero only for identical strings "levenshtein distance should be zero only for identical strings" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb) { s -> val resolver = createTestableAppResolver() val distance = resolver.levenshteinDistance(s, s) distance == 0 } } - + // Test triangle inequality for levenshtein distance "levenshtein distance should satisfy triangle inequality" { val stringArb = Arb.string(0..20, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb, stringArb) { s1, s2, s3 -> val resolver = createTestableAppResolver() val d12 = resolver.levenshteinDistance(s1, s2) @@ -85,46 +85,49 @@ class AppResolverPropertyTest : StringSpec({ d13 <= d12 + d23 } } - + // Test that similarity is always in range [0, 1] "similarity should always be in range 0 to 1" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { query, target -> val resolver = createTestableAppResolver() val similarity = resolver.calculateSimilarity(query.lowercase(), target.lowercase()) similarity >= 0.0 && similarity <= 1.0 } } - + // Test that contains match has higher similarity than fuzzy match "contains match should have higher similarity than non-contains fuzzy match" { val prefixArb = Arb.string(1..10, Codepoint.alphanumeric()) val suffixArb = Arb.string(1..10, Codepoint.alphanumeric()) val queryArb = Arb.string(3..15, Codepoint.alphanumeric()) - + forAll(100, prefixArb, queryArb, suffixArb) { prefix, query, suffix -> val resolver = createTestableAppResolver() val normalizedQuery = query.lowercase() - + // Target that contains the query val containsTarget = (prefix + query + suffix).lowercase() - + // Target that doesn't contain the query (completely different) val differentTarget = "zzz${prefix}zzz".lowercase() - + + // Skip valid match cases in the "different" target + if (differentTarget.contains(normalizedQuery)) return@forAll true + val containsSimilarity = resolver.calculateSimilarity(normalizedQuery, containsTarget) val differentSimilarity = resolver.calculateSimilarity(normalizedQuery, differentTarget) - + // Contains match should score higher (unless different target happens to be similar) containsSimilarity >= differentSimilarity || differentSimilarity < AppResolver.MIN_SIMILARITY_THRESHOLD } } - + // Test that levenshtein distance to empty string equals string length "levenshtein distance to empty string should equal string length" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb) { s -> val resolver = createTestableAppResolver() val distanceToEmpty = resolver.levenshteinDistance(s, "") @@ -132,19 +135,19 @@ class AppResolverPropertyTest : StringSpec({ distanceToEmpty == s.length && distanceFromEmpty == s.length } } - + // Test that startsWith match has high similarity "startsWith match should have high similarity" { val queryArb = Arb.string(3..15, Codepoint.alphanumeric()) val suffixArb = Arb.string(1..10, Codepoint.alphanumeric()) - + forAll(100, queryArb, suffixArb) { query, suffix -> val resolver = createTestableAppResolver() val normalizedQuery = query.lowercase() val target = (query + suffix).lowercase() - + val similarity = resolver.calculateSimilarity(normalizedQuery, target) - + // StartsWith should have similarity >= 0.75 similarity >= 0.75 } @@ -153,7 +156,7 @@ class AppResolverPropertyTest : StringSpec({ /** * Creates a testable AppResolver instance. - * + * * Since AppResolver requires PackageManager which is an Android system service, * we create a minimal mock that allows us to test the core similarity logic. */ @@ -164,12 +167,12 @@ private fun createTestableAppResolver(): TestableAppResolver { /** * A testable version of AppResolver that exposes the internal similarity * calculation methods for property-based testing. - * + * * This class duplicates the core logic from AppResolver to enable testing * without Android framework dependencies. */ class TestableAppResolver { - + /** * Calculates the similarity between two strings using a combination of: * 1. Exact match (highest priority) @@ -182,57 +185,57 @@ class TestableAppResolver { if (query == target) { return 1.0 } - + // Target contains query exactly if (target.contains(query)) { val coverageScore = query.length.toDouble() / target.length return 0.8 + (coverageScore * 0.15) } - + // Target starts with query if (target.startsWith(query)) { val coverageScore = query.length.toDouble() / target.length return 0.75 + (coverageScore * 0.15) } - + // Query starts with target if (query.startsWith(target)) { val coverageScore = target.length.toDouble() / query.length return 0.7 + (coverageScore * 0.15) } - + // Fuzzy matching using Levenshtein distance val distance = levenshteinDistance(query, target) val maxLength = maxOf(query.length, target.length) - + if (maxLength == 0) { return 0.0 } - + val similarity = 1.0 - (distance.toDouble() / maxLength) return similarity * 0.7 } - + /** * Calculates the Levenshtein distance between two strings. */ fun levenshteinDistance(s1: String, s2: String): Int { val m = s1.length val n = s2.length - + if (m == 0) return n if (n == 0) return m - + val dp = Array(m + 1) { IntArray(n + 1) } - + for (i in 0..m) { dp[i][0] = i } - + for (j in 0..n) { dp[0][j] = j } - + for (i in 1..m) { for (j in 1..n) { val cost = if (s1[i - 1] == s2[j - 1]) 0 else 1 @@ -243,10 +246,10 @@ class TestableAppResolver { ) } } - + return dp[m][n] } - + companion object { const val MIN_SIMILARITY_THRESHOLD = 0.3 } From 9a50b0c657feafec227fbe0bac568337a25eaf42 Mon Sep 17 00:00:00 2001 From: Repobor Date: Mon, 29 Dec 2025 22:52:37 +0800 Subject: [PATCH 6/6] fix: resolve keyboard enable failure caused by package name mismatch --- .../autoglm/input/AutoGLMKeyboardService.kt | 20 ++++++++-------- .../kevinluo/autoglm/input/KeyboardHelper.kt | 23 +++++++++++++------ .../autoglm/input/TextInputManager.kt | 17 +++++++------- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt b/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt index 484a688..7dbf7ab 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt @@ -47,7 +47,7 @@ class AutoGLMKeyboardService : InputMethodService() { private val inputReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context, intent: Intent) { Logger.d(TAG, "Received broadcast: ${intent.action}") - + when (intent.action) { ACTION_INPUT_TEXT, ACTION_INPUT_B64 -> { handleInputText(intent) @@ -90,13 +90,13 @@ class AutoGLMKeyboardService : InputMethodService() { */ override fun onCreateInputView(): View { Logger.d(TAG, "onCreateInputView called") - + // Register receiver when input view is created registerInputReceiver() - + // Create a minimal status view val view = layoutInflater.inflate(R.layout.keyboard_autoglm, null) - + return view } @@ -122,7 +122,7 @@ class AutoGLMKeyboardService : InputMethodService() { override fun onStartInputView(info: EditorInfo?, restarting: Boolean) { super.onStartInputView(info, restarting) Logger.d(TAG, "onStartInputView: restarting=$restarting") - + // Ensure receiver is registered registerInputReceiver() } @@ -165,7 +165,7 @@ class AutoGLMKeyboardService : InputMethodService() { try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - registerReceiver(inputReceiver, filter, Context.RECEIVER_EXPORTED) + registerReceiver(inputReceiver, filter, RECEIVER_EXPORTED) } else { @Suppress("UnspecifiedRegisterReceiverFlag") registerReceiver(inputReceiver, filter) @@ -211,7 +211,7 @@ class AutoGLMKeyboardService : InputMethodService() { val decodedBytes = Base64.decode(encodedText, Base64.DEFAULT) val text = String(decodedBytes, Charsets.UTF_8) Logger.d(TAG, "Decoded text: '${text.take(50)}${if (text.length > 50) "..." else ""}'") - + commitText(text) } catch (e: Exception) { Logger.e(TAG, "Failed to decode Base64 text", e) @@ -240,7 +240,7 @@ class AutoGLMKeyboardService : InputMethodService() { */ private fun handleClearText() { Logger.d(TAG, "Clearing text") - + val ic = currentInputConnection ?: run { Logger.w(TAG, "No input connection for clear text") return @@ -249,10 +249,10 @@ class AutoGLMKeyboardService : InputMethodService() { try { // Perform select all ic.performContextMenuAction(android.R.id.selectAll) - + // Delete selected text by committing empty string ic.commitText("", 0) - + Logger.d(TAG, "Text cleared successfully") } catch (e: Exception) { Logger.e(TAG, "Failed to clear text", e) diff --git a/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt b/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt index 9a754ee..816c20e 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt @@ -4,7 +4,9 @@ import android.content.Context import android.content.Intent import android.provider.Settings import android.view.inputmethod.InputMethodManager +import com.kevinluo.autoglm.BuildConfig import com.kevinluo.autoglm.util.Logger +import kotlin.coroutines.coroutineContext /** * Helper class for keyboard-related operations. @@ -16,17 +18,24 @@ import com.kevinluo.autoglm.util.Logger object KeyboardHelper { private const val TAG = "KeyboardHelper" - + /** AutoGLM package name. */ - private const val PACKAGE_NAME = "com.kevinluo.autoglm" - + private const val PACKAGE_NAME = BuildConfig.APPLICATION_ID + /** AutoGLM Keyboard IME ID (Android system format). */ - const val IME_ID = "$PACKAGE_NAME/.input.AutoGLMKeyboardService" - + + val IME_ID = "${BuildConfig.APPLICATION_ID}/${AutoGLMKeyboardService::class.java.name}" + + fun getImeId(context: Context): String { + val pkg = context.packageName + val service = AutoGLMKeyboardService::class.java.name + return "$pkg/$service" + } + /** * Checks if the given IME ID belongs to AutoGLM Keyboard. */ - fun isAutoGLMKeyboard(imeId: String): Boolean = + fun isAutoGLMKeyboard(imeId: String): Boolean = imeId.startsWith("$PACKAGE_NAME/") /** @@ -53,7 +62,7 @@ object KeyboardHelper { for (ime in enabledInputMethods) { Logger.d(TAG, "Found IME: package=${ime.packageName}, service=${ime.serviceName}") - if (ime.packageName == PACKAGE_NAME && + if (ime.packageName.contains(PACKAGE_NAME) && ime.serviceName.endsWith(".AutoGLMKeyboardService")) { Logger.d(TAG, "AutoGLM Keyboard is enabled") return KeyboardStatus.ENABLED diff --git a/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt b/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt index c9cf994..802ca83 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt @@ -1,6 +1,7 @@ package com.kevinluo.autoglm.input import android.util.Base64 +import com.kevinluo.autoglm.BuildConfig import com.kevinluo.autoglm.IUserService import com.kevinluo.autoglm.util.Logger import kotlinx.coroutines.Dispatchers @@ -36,7 +37,7 @@ class TextInputManager(private val userService: IUserService) { /** Cached original IME for restoration after text input. */ private var originalIme: String? = null - + /** * Types text into the currently focused input field. * @@ -112,11 +113,11 @@ class TextInputManager(private val userService: IUserService) { // Save original IME originalIme = currentIme Logger.d(TAG, "Saved original IME: $originalIme") - + // List all enabled IMEs to debug val enabledImes = shell("ime list -s") Logger.d(TAG, "Enabled IMEs:\n$enabledImes") - + // Get the IME ID val imeId = KeyboardHelper.IME_ID Logger.d(TAG, "AutoGLM Keyboard IME ID: $imeId") @@ -153,7 +154,7 @@ class TextInputManager(private val userService: IUserService) { val newIme = getCurrentIme() return KeyboardHelper.isAutoGLMKeyboard(newIme) } - + /** * Gets the current default input method. * @@ -173,7 +174,7 @@ class TextInputManager(private val userService: IUserService) { */ private fun clearText(): String { Logger.d(TAG, "Clearing text") - return shell("am broadcast -a $ACTION_CLEAR_TEXT -p $PACKAGE_NAME") + return shell("am broadcast -a $ACTION_CLEAR_TEXT -p $PACKAGE_NAME --user current") } /** @@ -192,7 +193,7 @@ class TextInputManager(private val userService: IUserService) { private fun inputTextViaB64(text: String): String { val encoded = Base64.encodeToString(text.toByteArray(Charsets.UTF_8), Base64.NO_WRAP) Logger.d(TAG, "Input text via B64: '$text' -> '$encoded'") - return shell("am broadcast -a $ACTION_INPUT_B64 -p $PACKAGE_NAME --es msg '$encoded'") + return shell("am broadcast -a $ACTION_INPUT_B64 -p $PACKAGE_NAME --es msg '$encoded' --user current") } /** @@ -232,9 +233,9 @@ class TextInputManager(private val userService: IUserService) { // Broadcast actions private const val ACTION_INPUT_B64 = "ADB_INPUT_B64" private const val ACTION_CLEAR_TEXT = "ADB_CLEAR_TEXT" - + // Package name - private const val PACKAGE_NAME = "com.kevinluo.autoglm" + private const val PACKAGE_NAME = BuildConfig.APPLICATION_ID // Timing constants (increased for stability) // Wait after switching keyboard to ensure it's fully active