chore: land experimental tools (#34503)

Author: pavelfeldman

package-lock.json

@@ -62,6 +62,11 @@ export class ElementHandle<T extends Node = Node> extends JSHandle<T> implements
     return Frame.fromNullable((await this._elementChannel.contentFrame()).frame);
   }
 
+  async _generateLocatorString(): Promise<string | null> {
+    const value = (await this._elementChannel.generateLocatorString()).value;
+    return value === undefined ? null : value;
+  }
+
   async getAttribute(name: string): Promise<string | null> {
     const value = (await this._elementChannel.getAttribute({ name })).value;
     return value === undefined ? null : value;

packages/playwright-core/src/client/elementHandle.ts

@@ -236,6 +236,10 @@ export class Locator implements api.Locator {
     return await this._frame._queryCount(this._selector);
   }
 
+  async _generateLocatorString(): Promise<string | null> {
+    return await this._withElement(h => h._generateLocatorString());
+  }
+
   async getAttribute(name: string, options?: TimeoutOptions): Promise<string | null> {
     return await this._frame.getAttribute(this._selector, name, { strict: true, ...options });
   }
@@ -288,8 +292,8 @@ export class Locator implements api.Locator {
     return await this._withElement((h, timeout) => h.screenshot({ ...options, timeout }), options.timeout);
   }
 
-  async ariaSnapshot(options?: TimeoutOptions): Promise<string> {
-    const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });
+  async ariaSnapshot(options?: { _id?: boolean, _mode?: 'raw' | 'regex' } & TimeoutOptions): Promise<string> {
+    const result = await this._frame._channel.ariaSnapshot({ ...options, id: options?._id, mode: options?._mode, selector: this._selector });
     return result.snapshot;
   }
 

packages/playwright-core/src/client/locator.ts

@@ -1429,6 +1429,8 @@ scheme.FrameAddStyleTagResult = tObject({
 });
 scheme.FrameAriaSnapshotParams = tObject({
   selector: tString,
+  id: tOptional(tBoolean),
+  mode: tOptional(tEnum(['raw', 'regex'])),
   timeout: tOptional(tNumber),
 });
 scheme.FrameAriaSnapshotResult = tObject({
@@ -1925,6 +1927,10 @@ scheme.ElementHandleFillParams = tObject({
 scheme.ElementHandleFillResult = tOptional(tObject({}));
 scheme.ElementHandleFocusParams = tOptional(tObject({}));
 scheme.ElementHandleFocusResult = tOptional(tObject({}));
+scheme.ElementHandleGenerateLocatorStringParams = tOptional(tObject({}));
+scheme.ElementHandleGenerateLocatorStringResult = tObject({
+  value: tOptional(tString),
+});
 scheme.ElementHandleGetAttributeParams = tObject({
   name: tString,
 });

packages/playwright-core/src/protocol/validator.ts

@@ -63,6 +63,10 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements chann
     return { frame: frame ? FrameDispatcher.from(this._browserContextDispatcher(), frame) : undefined };
   }
 
+  async generateLocatorString(params: channels.ElementHandleGenerateLocatorStringParams, metadata: CallMetadata): Promise<channels.ElementHandleGenerateLocatorStringResult> {
+    return { value: await this._elementHandle.generateLocatorString() };
+  }
+
   async getAttribute(params: channels.ElementHandleGetAttributeParams, metadata: CallMetadata): Promise<channels.ElementHandleGetAttributeResult> {
     const value = await this._elementHandle.getAttribute(metadata, params.name);
     return { value: value === null ? undefined : value };

packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts

@@ -28,7 +28,7 @@ import type { Progress } from './progress';
 import { ProgressController } from './progress';
 import type * as types from './types';
 import type { TimeoutOptions } from '../common/types';
-import { isUnderTest } from '../utils';
+import { asLocator, isUnderTest } from '../utils';
 import { prepareFilesForUpload } from './fileUploadUtils';
 
 export type InputFilesItems = {
@@ -185,6 +185,15 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
     return this._page._delegate.getContentFrame(this);
   }
 
+  async generateLocatorString(): Promise<string | undefined> {
+    const selector = await this.evaluateInUtility(async ([injected, node]) => {
+      return injected.generateSelectorSimple(node as unknown as Element);
+    }, {});
+    if (selector === 'error:notconnected')
+      return;
+    return asLocator('javascript', selector);
+  }
+
   async getAttribute(metadata: CallMetadata, name: string): Promise<string | null> {
     return this._frame.getAttribute(metadata, ':scope', name, {}, this);
   }
@@ -799,8 +808,8 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
     return this._page._delegate.getBoundingBox(this);
   }
 
-  async ariaSnapshot(): Promise<string> {
-    return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element), {});
+  async ariaSnapshot(options: { id?: boolean, mode?: 'raw' | 'regex' }): Promise<string> {
+    return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);
   }
 
   async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise<Buffer> {

packages/playwright-core/src/server/dom.ts

@@ -1408,10 +1408,10 @@ export class Frame extends SdkObject {
     });
   }
 
-  async ariaSnapshot(metadata: CallMetadata, selector: string, options: types.TimeoutOptions = {}): Promise<string> {
+  async ariaSnapshot(metadata: CallMetadata, selector: string, options: { id?: boolean, mode?: 'raw' | 'regex' } & types.TimeoutOptions = {}): Promise<string> {
     const controller = new ProgressController(metadata, this);
     return controller.run(async progress => {
-      return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot());
+      return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot(options));
     }, this._page._timeoutSettings.timeout(options));
   }
 

packages/playwright-core/src/server/frames.ts

@@ -85,10 +85,8 @@ class ConsoleAPI {
       inspect: (selector: string) => this._inspect(selector),
       selector: (element: Element) => this._selector(element),
       generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),
-      ariaSnapshot: (element?: Element) => {
-        const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body);
-        // eslint-disable-next-line no-console
-        console.log(snapshot);
+      ariaSnapshot: (element?: Element, options?: { id?: boolean }) => {
+        return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options);
       },
       resume: () => this._resume(),
       ...new Locator(injectedScript, ''),

packages/playwright-core/src/server/injected/consoleApi.ts

@@ -72,6 +72,7 @@ export class InjectedScript {
   // eslint-disable-next-line no-restricted-globals
   readonly window: Window & typeof globalThis;
   readonly document: Document;
+  private _ariaElementById: Map<number, Element> | undefined;
 
   // Recorder must use any external dependencies through InjectedScript.
   // Otherwise it will end up with a copy of all modules it uses, and any
@@ -130,6 +131,7 @@ export class InjectedScript {
     this._engines.set('internal:attr', this._createNamedAttributeEngine());
     this._engines.set('internal:testid', this._createNamedAttributeEngine());
     this._engines.set('internal:role', createRoleEngine(true));
+    this._engines.set('internal:aria-id', this._createAriaIdEngine());
 
     for (const { name, engine } of customEngines)
       this._engines.set(name, engine);
@@ -221,7 +223,8 @@ export class InjectedScript {
     if (node.nodeType !== Node.ELEMENT_NODE)
       throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
     const ariaSnapshot = generateAriaTree(node as Element);
-    return renderAriaTree(ariaSnapshot.root, options);
+    this._ariaElementById = ariaSnapshot.elements;
+    return renderAriaTree(ariaSnapshot.root, { ...options, ids: options?.id ? ariaSnapshot.ids : undefined });
   }
 
   ariaSnapshotAsObject(node: Node): AriaSnapshot {
@@ -609,6 +612,15 @@ export class InjectedScript {
     return result;
   }
 
+  _createAriaIdEngine() {
+    const queryAll = (root: SelectorRoot, selector: string): Element[] => {
+      const ariaId = parseInt(selector, 10);
+      const result = this._ariaElementById?.get(ariaId);
+      return result && result.isConnected ? [result] : [];
+    };
+    return { queryAll };
+  }
+
   elementState(node: Node, state: ElementStateWithoutStable): ElementStateQueryResult {
     const element = this.retarget(node, ['visible', 'hidden'].includes(state) ? 'none' : 'follow-label');
     if (!element || !element.isConnected) {

packages/playwright-core/src/server/injected/injectedScript.ts

@@ -39,7 +39,7 @@ export class Selectors {
       'internal:has', 'internal:has-not',
       'internal:has-text', 'internal:has-not-text',
       'internal:and', 'internal:or', 'internal:chain',
-      'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid',
+      'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', 'internal:aria-id'
     ]);
     this._builtinEnginesInMainWorld = new Set([
       '_react', '_vue',

packages/playwright-core/src/server/selectors.ts

@@ -0,0 +1,8 @@
+**/*
+README.md
+LICENSE
+!lib/*
+!browser.js
+!browser.d.ts
+!computer-20241022.js
+!computer-20241022.d.ts

packages/playwright-tools/.npmignore

@@ -0,0 +1 @@
+> **BEWARE** This package is EXPERIMENTAL and does not respect semver.

packages/playwright-tools/README.md

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import { ToolDeclaration, JSONSchemaType } from './types';
+
+export type ToolResult = {
+  error?: string;
+  code: Array<string>;
+  snapshot: string;
+}
+
+export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
+
+export const schema: ToolDeclaration[];
+export const call: ToolCall;
+export const snapshot: (page) => Promise<string>;

packages/playwright-tools/browser.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { schema, call, snapshot } = require('./lib/tools/browser');
+
+module.exports = { schema, call, snapshot };

packages/playwright-tools/browser.js

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type playwright from 'playwright';
+import { JSONSchemaType } from './types';
+
+export type ToolResult = {
+  output?: string;
+  error?: string;
+  base64_image?: string;
+};
+
+export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
+
+export const call: ToolCall;

packages/playwright-tools/computer-20241022.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { call } = require('./lib/tools/computer-20241022');
+
+module.exports = { call };

packages/playwright-tools/computer-20241022.js

@@ -0,0 +1,35 @@
+{
+  "name": "@playwright/experimental-tools",
+  "version": "1.51.0-next",
+  "description": "Playwright Tools for AI",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/playwright.git"
+  },
+  "homepage": "https://playwright.dev",
+  "engines": {
+    "node": ">=18"
+  },
+  "author": {
+    "name": "Microsoft Corporation"
+  },
+  "license": "Apache-2.0",
+  "exports": {
+    "./browser": {
+      "types": "./browser.d.ts",
+      "default": "./browser.js"
+    },
+    "./computer-20241022": {
+      "types": "./computer-20241022.d.ts",
+      "default": "./computer-20241022.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "dependencies": {
+    "playwright": "1.51.0-next"
+  },
+  "devDependencies": {
+    "@anthropic-ai/sdk": "^0.33.1",
+    "openai": "^4.79.1"
+  }
+}