From 1964d793f2186311884643fd7179cdf941ff5c02 Mon Sep 17 00:00:00 2001 From: Tyler Wickline Date: Sat, 14 Feb 2026 15:48:04 -0800 Subject: [PATCH 1/2] fix(browser): inject data-fgp-ref attributes during ARIA snapshot extraction The @eN element refs returned by browser.snapshot were not usable by interaction methods (click, fill, etc.) because nothing injected the corresponding data-fgp-ref attributes onto DOM elements. This adds attribute injection on both extraction paths: - CDP path: resolves BackendNodeId -> NodeId via DOM.describeNode, then sets data-fgp-ref via DOM.setAttributeValue - DOM fallback path: injects setAttribute('data-fgp-ref', 'eN') in the same JS loop that discovers interactive elements Old data-fgp-ref attributes are cleared at the start of each snapshot to prevent stale refs. --- src/browser/aria.rs | 97 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/src/browser/aria.rs b/src/browser/aria.rs index 783c49e..7a0abf7 100644 --- a/src/browser/aria.rs +++ b/src/browser/aria.rs @@ -1,32 +1,95 @@ //! ARIA tree extraction from Chrome DevTools Protocol. +//! +//! Extracts interactive elements and injects `data-fgp-ref` attributes onto +//! the DOM so that `@eN` refs returned in snapshots can be used by interaction +//! methods (click, fill, etc.) via `resolve_selector`. use anyhow::{Context, Result}; use chromiumoxide::cdp::browser_protocol::accessibility::{ AxNode as CdpAxNode, AxProperty, AxPropertyName, GetFullAxTreeParams, }; +use chromiumoxide::cdp::browser_protocol::dom::{DescribeNodeParams, SetAttributeValueParams}; use chromiumoxide::page::Page; use serde::Deserialize; use serde_json::Value as JsonValue; use crate::models::AriaNode; +/// Clear stale `data-fgp-ref` attributes from previous snapshots. +async fn clear_old_refs(page: &Page) { + let _ = page + .evaluate( + "document.querySelectorAll('[data-fgp-ref]').forEach(el => el.removeAttribute('data-fgp-ref'))", + ) + .await; +} + +/// Inject `data-fgp-ref` attributes onto DOM elements for CDP-extracted nodes. +/// +/// Uses `backend_dom_node_id` from each CDP AxNode to resolve the DOM element +/// via CDP, then sets `data-fgp-ref="eN"` so that `resolve_selector("@eN")` +/// can find it later. +async fn inject_refs_for_cdp_nodes(page: &Page, cdp_nodes: &[&CdpAxNode], aria_nodes: &[AriaNode]) { + for (cdp_node, aria_node) in cdp_nodes.iter().zip(aria_nodes.iter()) { + let backend_id = match cdp_node.backend_dom_node_id { + Some(id) => id, + None => continue, + }; + + // Strip the "@" prefix: "@e5" -> "e5" + let ref_value = &aria_node.ref_id[1..]; + + // Resolve BackendNodeId -> NodeId via DOM.describeNode + let describe_result = page + .execute( + DescribeNodeParams::builder() + .backend_node_id(backend_id) + .build(), + ) + .await; + + let node_id = match describe_result { + Ok(result) => result.node.node_id, + Err(_) => continue, + }; + + // Set the data-fgp-ref attribute on the DOM element + let _ = page + .execute(SetAttributeValueParams::new( + node_id, + "data-fgp-ref", + ref_value, + )) + .await; + } +} + /// Extract ARIA accessibility tree from page. pub async fn extract_aria_tree(page: &Page) -> Result> { let mut counter = 0; + // Clear stale refs from previous snapshots + clear_old_refs(page).await; + // Try CDP accessibility tree first if let Ok(response) = page.execute(GetFullAxTreeParams::default()).await { // Single-pass extraction - no clones, references only let capacity = response.nodes.len() / 4; // Most nodes filtered out + let mut included_cdp_nodes: Vec<&CdpAxNode> = Vec::with_capacity(capacity); let mut nodes = Vec::with_capacity(capacity); for node in &response.nodes { if is_interactive_node(node) || has_role_or_name(node) { + included_cdp_nodes.push(node); nodes.push(convert_node_ref(node, &mut counter)); } } if !nodes.is_empty() { + // Inject data-fgp-ref attributes onto the actual DOM elements + // so that resolve_selector("@eN") can find them later + inject_refs_for_cdp_nodes(page, &included_cdp_nodes, &nodes).await; + tracing::debug!( "Extracted {} nodes from CDP accessibility tree", nodes.len() @@ -36,6 +99,7 @@ pub async fn extract_aria_tree(page: &Page) -> Result> { } // Fallback to DOM traversal - more reliable on macOS + // This path injects data-fgp-ref attributes directly in the JS tracing::debug!("CDP accessibility tree empty, falling back to DOM traversal"); let nodes = extract_dom_interactives(page, &mut counter).await?; @@ -127,8 +191,12 @@ struct DomSnapshotNode { } async fn extract_dom_interactives(page: &Page, counter: &mut usize) -> Result> { - let script = r#"(() => { - const roleFor = (el) => { + // The JS discovers interactive elements, collects ARIA data, AND injects + // data-fgp-ref attributes in the same pass. The ref counter in JS starts + // at counter+1 to stay in sync with the Rust counter that increments below. + let script = format!( + r#"((startCounter) => {{ + const roleFor = (el) => {{ const explicit = el.getAttribute && el.getAttribute('role'); if (explicit) return explicit; const tag = el.tagName ? el.tagName.toLowerCase() : ''; @@ -142,7 +210,7 @@ async fn extract_dom_interactives(page: &Page, counter: &mut usize) -> Result Result { + }}; + const nameFor = (el) => {{ const label = el.getAttribute && el.getAttribute('aria-label'); if (label) return label; const alt = el.getAttribute && el.getAttribute('alt'); @@ -164,7 +232,7 @@ async fn extract_dom_interactives(page: &Page, counter: &mut usize) -> Result Result= 0, focused: document.activeElement === el, - }); - } + }}); + }} return nodes; - })()"#; + }})({})"#, + *counter + ); let dom_nodes: Vec = page .evaluate(script) From c29999c3f1f4c8667aa064d322efc32dce8d1e99 Mon Sep 17 00:00:00 2001 From: Tyler Wickline Date: Sat, 14 Feb 2026 16:00:19 -0800 Subject: [PATCH 2/2] test(browser): add integration tests for @eN ref resolution Adds two Chrome-dependent tests (run with `cargo test -- --ignored`): - snapshot_refs_are_findable: verifies that @eN refs from a snapshot resolve to actual DOM elements via data-fgp-ref attributes - snapshot_refs_refresh_on_second_call: verifies that a second snapshot re-injects valid refs that still resolve correctly --- src/browser/client.rs | 84 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/browser/client.rs b/src/browser/client.rs index 4b9f016..1f47c37 100644 --- a/src/browser/client.rs +++ b/src/browser/client.rs @@ -867,3 +867,87 @@ fn resolve_selector(selector: &str) -> String { fn count_nodes(nodes: &[crate::models::AriaNode]) -> usize { nodes.iter().map(|n| 1 + count_nodes(&n.children)).sum() } + +#[cfg(test)] +mod tests { + use super::*; + + /// End-to-end test: snapshot injects data-fgp-ref attributes, and @eN refs + /// resolve to findable elements. + /// + /// Requires Chrome — run with: cargo test -- --ignored snapshot_refs + #[tokio::test] + #[ignore] + async fn snapshot_refs_are_findable() { + let tmp = std::env::temp_dir().join("fgp-test-snapshot-refs"); + let client = BrowserClient::new(tmp, true) + .await + .expect("Failed to launch Chrome"); + + // Navigate to a page with known interactive elements + let html = r##"data:text/html, + + + A Link + "##; + client.navigate(html, None).await.unwrap(); + + // Take snapshot — this should inject data-fgp-ref attributes + let snapshot = client.snapshot(None).await.unwrap(); + assert!(!snapshot.nodes.is_empty(), "Snapshot should have nodes"); + + // Every node with an @eN ref should be findable via resolve_selector + let page = client.get_page(None).await.unwrap(); + let mut found = 0; + for node in &snapshot.nodes { + let css = resolve_selector(&node.ref_id); + if page.find_element(&css).await.is_ok() { + found += 1; + } + } + + assert!( + found > 0, + "At least one @eN ref should resolve to a DOM element, but none did. Nodes: {:?}", + snapshot.nodes.iter().map(|n| (&n.ref_id, &n.role, &n.name)).collect::>() + ); + } + + /// Verify that taking a second snapshot re-injects valid refs. + /// + /// Requires Chrome — run with: cargo test -- --ignored snapshot_refs_refresh + #[tokio::test] + #[ignore] + async fn snapshot_refs_refresh_on_second_call() { + let tmp = std::env::temp_dir().join("fgp-test-snapshot-refresh"); + let client = BrowserClient::new(tmp, true) + .await + .expect("Failed to launch Chrome"); + + let html = r##"data:text/html, + + A Link + "##; + client.navigate(html, None).await.unwrap(); + + // First snapshot + let snap1 = client.snapshot(None).await.unwrap(); + assert!(!snap1.nodes.is_empty()); + + // Second snapshot on the same page — refs should be refreshed + let snap2 = client.snapshot(None).await.unwrap(); + assert!(!snap2.nodes.is_empty()); + + // Refs from second snapshot should all be findable + let page = client.get_page(None).await.unwrap(); + for node in &snap2.nodes { + let css = resolve_selector(&node.ref_id); + assert!( + page.find_element(&css).await.is_ok(), + "Ref {} (role={}) should resolve after second snapshot", + node.ref_id, + node.role + ); + } + } +}