Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## [Unreleased]

### Added
- `--ensure-auth` flag for goto action - polls for auth completion at 2s intervals using URL-change heuristic instead of a static timed checkpoint. On success, closes headed browser, relaunches headless, and loads the original URL. Overrides `--no-auth-wall-detect` so auth detection runs even when wall detection is disabled
- Auto-detect authentication walls after goto navigation - uses three-heuristic detection (domain cookies, URL auth patterns, DOM login elements) and automatically opens headed checkpoint. Disable with `--no-auth-wall-detect` flag
- Smart default snapshot scoping - snapshots automatically scope to `<main>` element (then `[role="main"]`, fallback to `<body>`), reducing output size by excluding navigation, headers, and footers. Use `--snapshot-full` to capture full page body when needed
- `--snapshot-compact` flag for token-efficient LLM consumption - applies four transforms: link collapsing (merges link + /url child into `link "Title" -> /path`), heading inlining (merges heading with single link child), decorative image removal (strips img nodes with empty or single-char alt text), and duplicate URL dedup (removes second occurrence at same depth scope). Applied after `--snapshot-depth` and before `--snapshot-collapse` in the pipeline
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ web-ctl session end github

| Action | Usage | Returns |
|--------|-------|---------|
| `goto` | `run <s> goto <url> [--no-auth-wall-detect]` | `{ url, status, authWallDetected, checkpointCompleted, snapshot }` |
| `goto` | `run <s> goto <url> [--no-auth-wall-detect] [--ensure-auth]` | `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, snapshot }` |
| `snapshot` | `run <s> snapshot` | `{ url, snapshot }` |
| `click` | `run <s> click <sel> [--wait-stable]` | `{ url, clicked, snapshot }` |
| `click-wait` | `run <s> click-wait <sel> [--timeout]` | `{ url, clicked, settled, snapshot }` |
Expand Down Expand Up @@ -173,6 +173,7 @@ This eliminates the common click-snapshot-check loop that wastes agent turns on
| `--path <file>` | `screenshot` | Custom screenshot path (within session dir) |
| `--allow-evaluate` | `evaluate` | Required safety flag for JS execution |
| `--no-auth-wall-detect` | `goto` | Disable automatic auth wall detection and checkpoint opening |
| `--ensure-auth` | `goto` | Poll for auth completion instead of timed checkpoint; overrides `--no-auth-wall-detect` |
| `--snapshot-depth <N>` | Any action with snapshot | Limit ARIA tree depth (e.g. 3 for top 3 levels) |
| `--snapshot-selector <sel>` | Any action with snapshot | Scope snapshot to a DOM subtree |
| `--snapshot-max-lines <N>` | Any action with snapshot | Truncate snapshot to N lines |
Expand Down
2 changes: 1 addition & 1 deletion commands/web-ctl.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ node ${PLUGIN_ROOT}/scripts/web-ctl.js session end <name>
node ${PLUGIN_ROOT}/scripts/web-ctl.js session verify <name> --url <url>

# Browser actions
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url>
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url> [--ensure-auth]
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> snapshot
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> click <selector>
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> read <selector>
Expand Down
67 changes: 58 additions & 9 deletions scripts/web-ctl.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const BOOLEAN_FLAGS = new Set([
'--allow-evaluate', '--no-snapshot', '--wait-stable', '--vnc',
'--exact', '--accept', '--submit', '--dismiss', '--auto',
'--snapshot-collapse', '--snapshot-text-only', '--snapshot-compact',
'--snapshot-full', '--no-auth-wall-detect',
'--snapshot-full', '--no-auth-wall-detect', '--ensure-auth',
]);

function validateSessionName(name) {
Expand Down Expand Up @@ -951,7 +951,7 @@ async function runAction(sessionName, action, actionArgs, opts) {
if (!url) throw new Error('URL required: run <session> goto <url>');
validateUrl(url);
const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
if (!opts.noAuthWallDetect) {
if (opts.ensureAuth || !opts.noAuthWallDetect) {
const detection = await detectAuthWall(page, context, url);
if (detection.detected) {
console.warn('[WARN] Auth wall detected for ' + new URL(url).hostname);
Expand All @@ -963,13 +963,61 @@ async function runAction(sessionName, action, actionArgs, opts) {
page = headedBrowser.page;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
const ckTimeout = Math.min(opts.timeout ? parseInt(opts.timeout, 10) : 120, 3600) * 1000;
console.warn('[WARN] Checkpoint open for ' + (ckTimeout / 1000) + 's');
await new Promise(resolve => setTimeout(resolve, ckTimeout));
const snapshot = await getSnapshot(page, opts);
result = { url: page.url(), authWallDetected: true, checkpointCompleted: true,
...(snapshot != null && { snapshot }) };
break;
if (opts.ensureAuth) {
console.warn('[WARN] Waiting for auth completion (' + (ckTimeout / 1000) + 's timeout)');
const pollInterval = 2000;
const startTime = Date.now();
let authCompleted = false;
while (Date.now() - startTime < ckTimeout) {
await new Promise(resolve => setTimeout(resolve, pollInterval));
if (page.isClosed()) break;
try {
const authResult = await checkAuthSuccess(page, context, url, { loginUrl: url });
if (authResult.success) {
authCompleted = true;
break;
}
} catch { /* page may have navigated - retry next poll */ }
}
if (authCompleted) {
await closeBrowser(sessionName, context);
try {
const headlessBrowser = await launchBrowser(sessionName, { headless: true });
context = headlessBrowser.context;
page = headlessBrowser.page;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
const snapshot = await getSnapshot(page, opts);
result = { url: page.url(), authWallDetected: true, ensureAuthCompleted: true,
...(snapshot != null && { snapshot }) };
} catch (relaunchErr) {
result = { url, authWallDetected: true, ensureAuthCompleted: true,
message: 'Auth completed but headless reload failed: ' + relaunchErr.message };
context = null;
page = null;
}
break;
} else {
try { await closeBrowser(sessionName, context); } catch { /* already closed */ }
result = { url, authWallDetected: true, ensureAuthCompleted: false,
message: 'Auth did not complete within timeout' };
context = null;
page = null;
break;
}
} else {
console.warn('[WARN] Checkpoint open for ' + (ckTimeout / 1000) + 's');
await new Promise(resolve => setTimeout(resolve, ckTimeout));
const snapshot = await getSnapshot(page, opts);
result = { url: page.url(), authWallDetected: true, checkpointCompleted: true,
...(snapshot != null && { snapshot }) };
break;
}
} else {
if (opts.ensureAuth) {
result = { url: page.url(), authWallDetected: true, ensureAuthCompleted: false,
message: 'Auth wall detected but no display available for headed browser.' };
break;
}
const snapshot = await getSnapshot(page, opts);
result = { url: page.url(), authWallDetected: true, checkpointCompleted: false,
message: 'Auth wall detected but no display for headed checkpoint.',
Expand Down Expand Up @@ -1138,7 +1186,7 @@ async function runAction(sessionName, action, actionArgs, opts) {
}
} catch { /* ignore - page may have closed before URL read */ }

await closeBrowser(sessionName, context);
if (context) await closeBrowser(sessionName, context);
sessionStore.unlockSession(sessionName);
output({ ok: true, command: `run ${action}`, session: sessionName, ...(autoCreated && { autoCreated: true }), result });

Expand Down Expand Up @@ -1202,6 +1250,7 @@ Session commands:

Run actions:
goto <url> Navigate to URL
[--ensure-auth] Poll for auth completion instead of timed checkpoint
snapshot Get accessibility tree
click <selector> Click element
[--wait-stable] Wait for DOM + network to settle after click
Expand Down
6 changes: 4 additions & 2 deletions skills/web-browse/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ Safe practice: always double-quote URL arguments.
### goto - Navigate to URL

```bash
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url> [--no-auth-wall-detect]
node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url> [--no-auth-wall-detect] [--ensure-auth]
```

Navigates to a URL and automatically detects authentication walls using a three-heuristic detection system:
Expand All @@ -58,7 +58,9 @@ When an authentication wall is detected, the tool automatically opens a headed c

Use `--no-auth-wall-detect` to disable this automatic detection and skip the checkpoint, navigating headlessly without waiting for user interaction.

Returns: `{ url, status, authWallDetected, checkpointCompleted, snapshot }`
Use `--ensure-auth` to actively poll for authentication completion instead of a timed checkpoint. When set, the headed browser polls with `checkAuthSuccess` at 2-second intervals using the URL-change heuristic. On success, the headed browser closes, a headless browser relaunches, and the original URL is loaded. On timeout, returns `ensureAuthCompleted: false`. This flag overrides `--no-auth-wall-detect`.

Returns: `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, snapshot }`

### snapshot - Get Accessibility Tree

Expand Down
238 changes: 238 additions & 0 deletions tests/web-ctl-actions.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -759,3 +759,241 @@ describe('auth wall detection in goto', () => {
);
});
});

describe('--ensure-auth flag', () => {
const fs = require('fs');
const path = require('path');
const webCtlSource = fs.readFileSync(
path.join(__dirname, '..', 'scripts', 'web-ctl.js'),
'utf8'
);

it('BOOLEAN_FLAGS includes --ensure-auth', () => {
assert.ok(
webCtlSource.includes("'--ensure-auth'"),
'--ensure-auth should be in BOOLEAN_FLAGS'
);
});

it('help text contains --ensure-auth flag', () => {
assert.ok(
webCtlSource.includes('--ensure-auth'),
'help text should document --ensure-auth'
);
});

it('goto case references opts.ensureAuth', () => {
assert.ok(
webCtlSource.includes('opts.ensureAuth'),
'goto case should check ensureAuth flag'
);
});

it('goto case calls checkAuthSuccess when ensureAuth is set', () => {
// Verify checkAuthSuccess is called within the ensureAuth block
const ensureAuthIdx = webCtlSource.indexOf('if (opts.ensureAuth)');
const checkAuthIdx = webCtlSource.indexOf('checkAuthSuccess(page, context, url, { loginUrl: url })', ensureAuthIdx);
assert.ok(ensureAuthIdx > 0, 'opts.ensureAuth guard should exist');
assert.ok(checkAuthIdx > ensureAuthIdx, 'checkAuthSuccess should be called after ensureAuth check');
});

it('result includes ensureAuthCompleted on success path', () => {
assert.ok(
webCtlSource.includes('ensureAuthCompleted: true'),
'success result should include ensureAuthCompleted: true'
);
});

it('result includes ensureAuthCompleted on timeout path', () => {
assert.ok(
webCtlSource.includes('ensureAuthCompleted: false'),
'timeout result should include ensureAuthCompleted: false'
);
});

it('timeout result includes descriptive message', () => {
assert.ok(
webCtlSource.includes('Auth did not complete within timeout'),
'timeout path should include descriptive message'
);
});

it('no-display path returns ensureAuthCompleted false', () => {
assert.ok(
webCtlSource.includes('no display available for headed browser'),
'no-display path should include descriptive message when ensureAuth is set'
);
});

it('ensureAuth overrides noAuthWallDetect in guard condition', () => {
assert.ok(
webCtlSource.includes('opts.ensureAuth || !opts.noAuthWallDetect'),
'guard should allow ensureAuth to override noAuthWallDetect'
);
});

it('relaunches headless browser after auth success', () => {
const ensureAuthIdx = webCtlSource.indexOf('if (opts.ensureAuth)');
const headlessRelaunch = webCtlSource.indexOf("launchBrowser(sessionName, { headless: true })", ensureAuthIdx);
assert.ok(headlessRelaunch > ensureAuthIdx, 'should relaunch headless browser after successful auth');
});

it('polls at 2s intervals', () => {
assert.ok(
webCtlSource.includes('const pollInterval = 2000'),
'poll interval should be 2000ms'
);
});

it('checks page.isClosed() before polling', () => {
const ensureAuthIdx = webCtlSource.indexOf('if (opts.ensureAuth)');
const isClosedIdx = webCtlSource.indexOf('page.isClosed()', ensureAuthIdx);
assert.ok(isClosedIdx > ensureAuthIdx, 'should check page.isClosed() in polling loop');
});

it('wraps checkAuthSuccess in try-catch during polling', () => {
const ensureAuthIdx = webCtlSource.indexOf('if (opts.ensureAuth)');
const nextCheckpoint = webCtlSource.indexOf('} else {', ensureAuthIdx + 200);
const pollBlock = webCtlSource.slice(ensureAuthIdx, nextCheckpoint);
assert.ok(
pollBlock.includes('try {') && pollBlock.includes('checkAuthSuccess'),
'checkAuthSuccess should be wrapped in try-catch within polling loop'
);
});

it('guards closeBrowser with context null check on normal exit', () => {
assert.ok(
webCtlSource.includes('if (context) await closeBrowser(sessionName, context)'),
'normal exit should guard closeBrowser with context null check'
);
});

it('nullifies context and page on timeout path', () => {
const timeoutIdx = webCtlSource.indexOf('Auth did not complete within timeout');
const nextBreak = webCtlSource.indexOf('break;', timeoutIdx);
const block = webCtlSource.slice(timeoutIdx, nextBreak);
assert.ok(block.includes('context = null'), 'timeout path should set context to null');
assert.ok(block.includes('page = null'), 'timeout path should set page to null');
});

it('breaks polling loop when page is closed', () => {
const ensureAuthIdx = webCtlSource.indexOf('if (opts.ensureAuth)');
const pollBlock = webCtlSource.slice(ensureAuthIdx, ensureAuthIdx + 600);
assert.ok(
pollBlock.includes('if (page.isClosed()) break'),
'polling loop should break when page is closed'
);
});

it('wraps headless relaunch in try-catch on success path', () => {
const authCompletedIdx = webCtlSource.indexOf('if (authCompleted)');
const elseIdx = webCtlSource.indexOf('} else {', authCompletedIdx + 50);
const successBlock = webCtlSource.slice(authCompletedIdx, elseIdx);
assert.ok(
successBlock.includes('try {') && successBlock.includes('launchBrowser'),
'success path should wrap headless relaunch in try-catch'
);
});

it('wraps closeBrowser in try-catch on timeout path', () => {
const elseIdx = webCtlSource.indexOf('Auth did not complete within timeout');
const beforeTimeout = webCtlSource.slice(elseIdx - 300, elseIdx);
assert.ok(
beforeTimeout.includes('try { await closeBrowser'),
'timeout path should wrap closeBrowser in try-catch'
);
});
});

describe('--ensure-auth flag parsing', () => {
const BOOLEAN_FLAGS = new Set([
'--allow-evaluate', '--no-snapshot', '--wait-stable', '--vnc',
'--exact', '--accept', '--submit', '--dismiss', '--auto',
'--snapshot-collapse', '--snapshot-text-only', '--snapshot-compact',
'--snapshot-full', '--no-auth-wall-detect', '--ensure-auth',
]);

function parseOptions(args) {
const opts = {};
for (let i = 0; i < args.length; i++) {
if (args[i].startsWith('--')) {
const key = args[i].slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
const next = args[i + 1];
if (next && !next.startsWith('--') && !BOOLEAN_FLAGS.has(args[i])) {
opts[key] = next;
i++;
} else {
opts[key] = true;
}
}
}
return opts;
}

it('--ensure-auth parses as boolean true', () => {
const opts = parseOptions(['--ensure-auth']);
assert.equal(opts.ensureAuth, true);
});

it('--ensure-auth does not consume next positional arg', () => {
const opts = parseOptions(['--ensure-auth', 'https://example.com']);
assert.equal(opts.ensureAuth, true);
assert.equal(opts['https://example.com'], undefined);
});

it('--ensure-auth works alongside --timeout', () => {
const opts = parseOptions(['--ensure-auth', '--timeout', '60']);
assert.equal(opts.ensureAuth, true);
assert.equal(opts.timeout, '60');
});

it('--ensure-auth works alongside --no-auth-wall-detect', () => {
const opts = parseOptions(['--ensure-auth', '--no-auth-wall-detect']);
assert.equal(opts.ensureAuth, true);
assert.equal(opts.noAuthWallDetect, true);
});
});

describe('--ensure-auth CLI integration', () => {
const { beforeEach, afterEach } = require('node:test');
const fs = require('fs');
const path = require('path');
const os = require('os');
const { execFileSync } = require('child_process');

let tmpDir;

beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'web-ctl-ensure-auth-'));
});

afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});

function runCliSafe(...args) {
try {
const result = execFileSync(process.execPath, [
path.join(__dirname, '..', 'scripts', 'web-ctl.js'),
...args
], {
env: { ...process.env, AI_STATE_DIR: tmpDir },
encoding: 'utf8',
timeout: 15000
});
return JSON.parse(result);
} catch (err) {
if (err.stdout) {
try { return JSON.parse(err.stdout); } catch { /* fall through */ }
}
throw err;
}
}

it('goto with --ensure-auth against example.com runs without error', () => {
const result = runCliSafe('run', 'authtest', 'goto', 'https://example.com', '--ensure-auth');
assert.ok(result, 'should return a result');
assert.notEqual(result.error, 'invalid_flag',
'--ensure-auth should not cause an invalid flag error');
});
});
Loading