Skip to content

fix: allow first/last as modifiers after role/text locators in find command #389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
liuxiaopai-ai wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: allow first/last as modifiers after role/text locators in find command #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 203 additions & 19 deletions cli/src/commands.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1029,21 +1029,48 @@ fn parse_find(rest: &[&str], id: &str) -> Result<Value, ParseError> {
let value = rest.get(1).ok_or_else(|| ParseError::MissingArguments {
context: format!("find {}", locator),
usage: match *locator {
"role" => "find role <role> [action] [--name <name>] [--exact]",
"text" => "find text <text> [action] [--exact]",
"label" => "find label <label> [action] [text] [--exact]",
"placeholder" => "find placeholder <text> [action] [text] [--exact]",
"alt" => "find alt <text> [action] [--exact]",
"title" => "find title <text> [action] [--exact]",
"testid" => "find testid <id> [action] [text]",
"role" => "find role <role> [first|last|nth <n>] [action] [--name <name>] [--exact]",
"text" => "find text <text> [first|last|nth <n>] [action] [--exact]",
"label" => "find label <label> [first|last|nth <n>] [action] [text] [--exact]",
"placeholder" => "find placeholder <text> [first|last|nth <n>] [action] [text] [--exact]",
"alt" => "find alt <text> [first|last|nth <n>] [action] [--exact]",
"title" => "find title <text> [first|last|nth <n>] [action] [--exact]",
"testid" => "find testid <id> [first|last|nth <n>] [action] [text]",
"first" => "find first <selector> [action] [text]",
"last" => "find last <selector> [action] [text]",
_ => "find <locator> <value> [action] [text]",
},
})?;
let subaction = rest.get(2).unwrap_or(&"click");
let fill_value = if rest.len() > 3 {
Some(rest[3..].join(" "))

// Check if next token after value is a modifier (first/last/nth)
// For semantic locators (role, text, label, etc.), first/last/nth can be
// used as modifiers: e.g. `find role spinbutton first fill '20'`
let modifier_token = rest.get(2).map(|s| *s);
let (index, args_offset) = match (*locator, modifier_token) {
// When locator is already first/last, don't look for modifier
("first", _) | ("last", _) => (None, 0),
// first/last/nth as modifier after a semantic locator
(_, Some("first")) => (Some(0i32), 1),
(_, Some("last")) => (Some(-1i32), 1),
(_, Some("nth")) => {
let nth_val = rest.get(3).ok_or_else(|| ParseError::MissingArguments {
context: format!("find {} {} nth", locator, value),
usage: "find <locator> <value> nth <index> [action] [text]",
})?;
let idx = nth_val.parse::<i32>().map_err(|_| ParseError::MissingArguments {
context: format!("find {} {} nth", locator, value),
usage: "find <locator> <value> nth <index> [action] [text]",
})?;
(Some(idx), 2)
}
_ => (None, 0),
};

let subaction_pos = 2 + args_offset;
let subaction = rest.get(subaction_pos).unwrap_or(&"click");
let fill_start = subaction_pos + 1;
let fill_value = if rest.len() > fill_start {
Some(rest[fill_start..].join(" "))
} else {
None
};
Expand All @@ -1054,36 +1081,69 @@ fn parse_find(rest: &[&str], id: &str) -> Result<Value, ParseError> {
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"text" => {
let mut cmd = json!({ "id": id, "action": "getbytext", "text": value, "subaction": subaction, "exact": exact });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"text" => Ok(
json!({ "id": id, "action": "getbytext", "text": value, "subaction": subaction, "exact": exact }),
),
"label" => {
let mut cmd = json!({ "id": id, "action": "getbylabel", "label": value, "subaction": subaction, "exact": exact });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"placeholder" => {
let mut cmd = json!({ "id": id, "action": "getbyplaceholder", "placeholder": value, "subaction": subaction, "exact": exact });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"alt" => {
let mut cmd = json!({ "id": id, "action": "getbyalttext", "text": value, "subaction": subaction, "exact": exact });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"title" => {
let mut cmd = json!({ "id": id, "action": "getbytitle", "text": value, "subaction": subaction, "exact": exact });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"alt" => Ok(
json!({ "id": id, "action": "getbyalttext", "text": value, "subaction": subaction, "exact": exact }),
),
"title" => Ok(
json!({ "id": id, "action": "getbytitle", "text": value, "subaction": subaction, "exact": exact }),
),
"testid" => {
let mut cmd = json!({ "id": id, "action": "getbytestid", "testId": value, "subaction": subaction });
if let Some(v) = fill_value {
cmd["value"] = json!(v);
}
if let Some(idx) = index {
cmd["index"] = json!(idx);
}
Ok(cmd)
}
"first" => {
Expand Down Expand Up @@ -2285,6 +2345,130 @@ mod tests {
assert!(cmd.get("value").is_none());
}

// === find first/last as modifier tests (issue #364) ===

#[test]
fn test_find_role_first_modifier_click() {
let cmd =
parse_command(&args("find role spinbutton first click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbyrole");
assert_eq!(cmd["role"], "spinbutton");
assert_eq!(cmd["index"], 0);
assert_eq!(cmd["subaction"], "click");
assert!(cmd.get("value").is_none());
}

#[test]
fn test_find_role_first_modifier_fill() {
let cmd =
parse_command(&args("find role spinbutton first fill 20"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbyrole");
assert_eq!(cmd["role"], "spinbutton");
assert_eq!(cmd["index"], 0);
assert_eq!(cmd["subaction"], "fill");
assert_eq!(cmd["value"], "20");
}

#[test]
fn test_find_role_last_modifier() {
let cmd =
parse_command(&args("find role textbox last fill hello"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbyrole");
assert_eq!(cmd["role"], "textbox");
assert_eq!(cmd["index"], -1);
assert_eq!(cmd["subaction"], "fill");
assert_eq!(cmd["value"], "hello");
}

#[test]
fn test_find_text_first_modifier() {
let cmd =
parse_command(&args("find text Submit first click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbytext");
assert_eq!(cmd["text"], "Submit");
assert_eq!(cmd["index"], 0);
assert_eq!(cmd["subaction"], "click");
}

#[test]
fn test_find_label_last_modifier() {
let cmd =
parse_command(&args("find label Email last fill test@test.com"), &default_flags())
.unwrap();
assert_eq!(cmd["action"], "getbylabel");
assert_eq!(cmd["label"], "Email");
assert_eq!(cmd["index"], -1);
assert_eq!(cmd["subaction"], "fill");
assert_eq!(cmd["value"], "test@test.com");
}

#[test]
fn test_find_role_nth_modifier() {
let cmd =
parse_command(&args("find role listitem nth 3 click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbyrole");
assert_eq!(cmd["role"], "listitem");
assert_eq!(cmd["index"], 3);
assert_eq!(cmd["subaction"], "click");
}

#[test]
fn test_find_role_without_modifier_no_index() {
// Backward compat: find role spinbutton click (no modifier) should NOT have index
let cmd =
parse_command(&args("find role spinbutton click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "getbyrole");
assert_eq!(cmd["role"], "spinbutton");
assert_eq!(cmd["subaction"], "click");
assert!(cmd.get("index").is_none());
}

#[test]
fn test_find_first_standalone_still_works() {
// Backward compat: find first <selector> still works as before
let cmd = parse_command(&args("find first .item click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "nth");
assert_eq!(cmd["selector"], ".item");
assert_eq!(cmd["index"], 0);
assert_eq!(cmd["subaction"], "click");
}

#[test]
fn test_find_last_standalone_still_works() {
let cmd = parse_command(&args("find last .item click"), &default_flags()).unwrap();
assert_eq!(cmd["action"], "nth");
assert_eq!(cmd["selector"], ".item");
assert_eq!(cmd["index"], -1);
assert_eq!(cmd["subaction"], "click");
}

#[test]
fn test_find_placeholder_first_modifier() {
let cmd = parse_command(
&args("find placeholder Search first fill query"),
&default_flags(),
)
.unwrap();
assert_eq!(cmd["action"], "getbyplaceholder");
assert_eq!(cmd["placeholder"], "Search");
assert_eq!(cmd["index"], 0);
assert_eq!(cmd["subaction"], "fill");
assert_eq!(cmd["value"], "query");
}

#[test]
fn test_find_testid_last_modifier() {
let cmd = parse_command(
&args("find testid submit-btn last click"),
&default_flags(),
)
.unwrap();
assert_eq!(cmd["action"], "getbytestid");
assert_eq!(cmd["testId"], "submit-btn");
assert_eq!(cmd["index"], -1);
assert_eq!(cmd["subaction"], "click");
}

// === Download Tests ===

#[test]
Expand Down
10 changes: 9 additions & 1 deletion cli/src/output.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1043,7 +1043,7 @@ Examples:
r##"
agent-browser find - Find and interact with elements by locator

Usage: agent-browser find <locator> <value> [action] [text]
Usage: agent-browser find <locator> <value> [first|last|nth <n>] [action] [text]

Finds elements using semantic locators and optionally performs an action.

Expand All @@ -1059,6 +1059,11 @@ Locators:
last <selector> Last matching element
nth <index> <selector> Nth matching element (0-based)

Modifiers (chain after any locator to narrow results):
first Select the first match
last Select the last match
nth <index> Select the nth match (0-based)

Actions (default: click):
click, fill, type, hover, focus, check, uncheck

Expand All @@ -1078,6 +1083,9 @@ Examples:
agent-browser find testid "login-form" click
agent-browser find first "li.item" click
agent-browser find nth 2 ".card" hover
agent-browser find role spinbutton first fill "20"
agent-browser find text "Submit" last click
agent-browser find label "Email" nth 2 fill "user@example.com"
"##
}

Expand Down
38 changes: 31 additions & 7 deletions src/actions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,7 +887,10 @@ async function handleGetByRole(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByRole(command.role as any, { name: command.name });
let locator = page.getByRole(command.role as any, { name: command.name });
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}
Comment on lines +891 to +893
Copy link

@chatgpt-codex-connector chatgpt-codex-connector bot Feb 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Propagate semantic locator index through validation

These handlers now rely on command.index to apply .last()/.nth(), but the command validator in src/protocol.ts still does not define an index field for any getby* schema and dispatches result.data from safeParse. That means the new index emitted by the CLI is dropped before execution, so commands like find role spinbutton first click still target the unscoped locator and can fail when multiple matches exist.

Useful? React with 👍 / 👎.


switch (command.subaction) {
case 'click':
Expand All @@ -910,12 +913,18 @@ async function handleGetByText(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByText(command.text, { exact: command.exact });
let locator = page.getByText(command.text, { exact: command.exact });
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
await locator.click();
return successResponse(command.id, { clicked: true });
case 'fill':
await locator.fill(command.value ?? '');
return successResponse(command.id, { filled: true });
Comment on lines +925 to +927
Copy link

@chatgpt-codex-connector chatgpt-codex-connector bot Feb 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Accept getbytext fill in protocol schema

A fill path was added here for getbytext, but src/protocol.ts still validates getbytext.subaction as only click or hover. As a result, find text ... fill ... is rejected during command parsing and never reaches this branch, so the new behavior is effectively broken despite the type/handler changes.

Useful? React with 👍 / 👎.

case 'hover':
await locator.hover();
return successResponse(command.id, { hovered: true });
Expand All @@ -927,7 +936,10 @@ async function handleGetByLabel(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByLabel(command.label);
let locator = page.getByLabel(command.label);
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
Expand All @@ -947,7 +959,10 @@ async function handleGetByPlaceholder(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByPlaceholder(command.placeholder);
let locator = page.getByPlaceholder(command.placeholder);
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
Expand Down Expand Up @@ -1660,7 +1675,10 @@ async function handleGetByAltText(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByAltText(command.text, { exact: command.exact });
let locator = page.getByAltText(command.text, { exact: command.exact });
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
Expand All @@ -1677,7 +1695,10 @@ async function handleGetByTitle(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByTitle(command.text, { exact: command.exact });
let locator = page.getByTitle(command.text, { exact: command.exact });
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
Expand All @@ -1694,7 +1715,10 @@ async function handleGetByTestId(
browser: BrowserManager
): Promise<Response> {
const page = browser.getPage();
const locator = page.getByTestId(command.testId);
let locator = page.getByTestId(command.testId);
if (command.index !== undefined) {
locator = command.index === -1 ? locator.last() : locator.nth(command.index);
}

switch (command.subaction) {
case 'click':
Expand Down
Loading