From f4c9b089dbf226b918cbeb1563ee2775882eed30 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 26 Feb 2026 04:05:30 +0200 Subject: [PATCH 1/5] feat(stealth): deep anti-bot evasion for headless browsers CDP artifact removal, screen/viewport dimension spoofing, navigator.connection, WebRTC IP leak prevention, realistic viewport. --- scripts/browser-launcher.js | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/browser-launcher.js b/scripts/browser-launcher.js index d0bc6b2..2ddb4e9 100644 --- a/scripts/browser-launcher.js +++ b/scripts/browser-launcher.js @@ -75,10 +75,12 @@ async function launchBrowser(sessionName, options = {}) { const launchOptions = { headless, + viewport: { width: 1920, height: 1080 }, args: [ '--disable-blink-features=AutomationControlled', '--no-first-run', - '--no-default-browser-check' + '--no-default-browser-check', + '--window-size=1920,1080' ] }; @@ -146,6 +148,34 @@ async function launchBrowser(sessionName, options = {}) { } return origQuery(params); }; + + // Remove CDP detection artifacts (window.cdc_* variables) + for (const key of Object.keys(window)) { + if (/^cdc_/.test(key)) delete window[key]; + } + + // Screen dimensions (headless reports 0 for outerWidth/outerHeight) + Object.defineProperty(window, 'outerWidth', { get: () => window.innerWidth }); + Object.defineProperty(window, 'outerHeight', { get: () => window.innerHeight + 85 }); + Object.defineProperty(screen, 'availWidth', { get: () => screen.width }); + Object.defineProperty(screen, 'availHeight', { get: () => screen.height - 40 }); + + // navigator.connection (missing in some headless environments) + if (!navigator.connection) { + Object.defineProperty(navigator, 'connection', { + get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, saveData: false }) + }); + } + + // Prevent WebRTC local IP leak (fingerprinting signal) + if (window.RTCPeerConnection) { + const OrigRTC = window.RTCPeerConnection; + window.RTCPeerConnection = function(config, constraints) { + if (config && config.iceServers) config.iceServers = []; + return new OrigRTC(config, constraints); + }; + window.RTCPeerConnection.prototype = OrigRTC.prototype; + } }); // Get or create the first page From 7804951c6f972f5b890572758be8ad66bbd0dbd7 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 26 Feb 2026 04:05:38 +0200 Subject: [PATCH 2/5] feat(goto): auto headed fallback when content blocked in headless When content blocking is detected, automatically switches to headed browser to retrieve content. Disable with --no-auto-recover. --- scripts/web-ctl.js | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/scripts/web-ctl.js b/scripts/web-ctl.js index 943d131..c88b6dd 100755 --- a/scripts/web-ctl.js +++ b/scripts/web-ctl.js @@ -21,7 +21,7 @@ const BOOLEAN_FLAGS = new Set([ '--allow-evaluate', '--no-snapshot', '--wait-stable', '--vnc', '--exact', '--accept', '--submit', '--dismiss', '--auto', '--snapshot-collapse', '--snapshot-text-only', '--snapshot-compact', - '--snapshot-full', '--no-auth-wall-detect', '--no-content-block-detect', '--ensure-auth', '--wait-loaded', + '--snapshot-full', '--no-auth-wall-detect', '--no-content-block-detect', '--no-auto-recover', '--ensure-auth', '--wait-loaded', ]); function validateSessionName(name) { @@ -1096,6 +1096,39 @@ async function runAction(sessionName, action, actionArgs, opts) { contentBlockedIndicators: provider?.contentBlockedIndicators }); } + // Auto headed fallback when content is blocked + if (contentBlockResult?.detected && !opts.noAutoRecover) { + const headed = await canLaunchHeaded(); + if (headed) { + console.warn('[WARN] Content blocked in headless - falling back to headed browser'); + await closeBrowser(sessionName, context); + await new Promise(resolve => setTimeout(resolve, 500)); + const headedBrowser = await launchBrowser(sessionName, { headless: false }); + context = headedBrowser.context; + page = headedBrowser.page; + try { + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }); + if (opts.waitLoaded) { + await waitForLoaded(page, { timeout: loadedTimeout }); + } + const headedSnapshot = await getSnapshot(page, opts); + result = { + url: page.url(), + status: response ? response.status() : null, + contentBlocked: true, + headedFallback: true, + warning: 'content_blocked_headed_fallback', + suggestion: 'Content was blocked in headless mode. Retrieved via headed browser.', + ...(opts.waitLoaded && { waitLoaded: true }), + ...(headedSnapshot != null && { snapshot: headedSnapshot }) + }; + break; + } catch (fallbackErr) { + console.warn('[WARN] Headed fallback failed: ' + fallbackErr.message); + // Fall through to return the headless result with warning + } + } + } const snapshot = await getSnapshot(page, opts); result = { url: page.url(), @@ -1103,9 +1136,12 @@ async function runAction(sessionName, action, actionArgs, opts) { ...(opts.waitLoaded && { waitLoaded: true }), ...(contentBlockResult?.detected && { contentBlocked: true, + headedFallback: false, warning: 'content_blocked', contentBlockedReason: contentBlockResult.reason, - suggestion: "Site may be blocking headless browsers. Try: (1) authenticate with 'session auth --provider ', (2) use --ensure-auth for headed mode" + suggestion: opts.noAutoRecover + ? "Site may be blocking headless browsers. Try: (1) authenticate with 'session auth --provider ', (2) use --ensure-auth for headed mode" + : 'Content blocked and no display for headed fallback. Try: ssh -X or set DISPLAY.' }), ...(snapshot != null && { snapshot }) }; From b2d10950aca368553007eb7ab8e76d1510a8d269 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 26 Feb 2026 04:05:47 +0200 Subject: [PATCH 3/5] test: add stealth hardening and auto-recover fallback tests --- tests/browser-stealth.test.js | 144 ++++++++++++++++++++++++++++++++++ tests/web-ctl-actions.test.js | 48 ++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 tests/browser-stealth.test.js diff --git a/tests/browser-stealth.test.js b/tests/browser-stealth.test.js new file mode 100644 index 0000000..a8d50be --- /dev/null +++ b/tests/browser-stealth.test.js @@ -0,0 +1,144 @@ +'use strict'; + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); + +const launcherSource = fs.readFileSync( + path.join(__dirname, '..', 'scripts', 'browser-launcher.js'), + 'utf8' +); + +describe('browser stealth init script', () => { + + it('hides navigator.webdriver', () => { + assert.ok( + launcherSource.includes("navigator, 'webdriver'"), + 'should spoof navigator.webdriver' + ); + }); + + it('spoofs window.chrome object', () => { + assert.ok( + launcherSource.includes('window.chrome'), + 'should define window.chrome' + ); + }); + + it('spoofs navigator.plugins', () => { + assert.ok( + launcherSource.includes("navigator, 'plugins'"), + 'should spoof navigator.plugins' + ); + assert.ok( + launcherSource.includes('Chrome PDF Plugin'), + 'should include a realistic plugin' + ); + }); + + it('spoofs navigator.languages', () => { + assert.ok( + launcherSource.includes("navigator, 'languages'"), + 'should spoof navigator.languages' + ); + }); + + it('overrides WebGL renderer', () => { + assert.ok( + launcherSource.includes('UNMASKED_VENDOR_WEBGL'), + 'should override WebGL vendor' + ); + assert.ok( + launcherSource.includes('UNMASKED_RENDERER_WEBGL'), + 'should override WebGL renderer' + ); + }); + + it('overrides permissions.query', () => { + assert.ok( + launcherSource.includes('permissions.query'), + 'should override permissions.query' + ); + }); + + it('removes CDP detection artifacts', () => { + assert.ok( + launcherSource.includes('cdc_'), + 'should remove cdc_ variables' + ); + }); + + it('spoofs screen dimensions', () => { + assert.ok( + launcherSource.includes("'outerWidth'"), + 'should spoof window.outerWidth' + ); + assert.ok( + launcherSource.includes("'outerHeight'"), + 'should spoof window.outerHeight' + ); + assert.ok( + launcherSource.includes("'availWidth'"), + 'should spoof screen.availWidth' + ); + assert.ok( + launcherSource.includes("'availHeight'"), + 'should spoof screen.availHeight' + ); + }); + + it('spoofs navigator.connection', () => { + assert.ok( + launcherSource.includes('navigator.connection'), + 'should spoof navigator.connection' + ); + assert.ok( + launcherSource.includes("effectiveType: '4g'"), + 'should report 4g connection' + ); + }); + + it('prevents WebRTC IP leak', () => { + assert.ok( + launcherSource.includes('RTCPeerConnection'), + 'should override RTCPeerConnection' + ); + assert.ok( + launcherSource.includes('iceServers'), + 'should clear iceServers' + ); + }); +}); + +describe('browser launch options', () => { + + it('sets realistic viewport size', () => { + assert.ok( + launcherSource.includes('viewport:'), + 'should set viewport' + ); + assert.ok( + launcherSource.includes('1920'), + 'should use 1920 width' + ); + assert.ok( + launcherSource.includes('1080'), + 'should use 1080 height' + ); + }); + + it('sets window-size arg', () => { + assert.ok( + launcherSource.includes('--window-size=1920,1080'), + 'should set --window-size arg' + ); + }); + + it('disables automation controlled features', () => { + assert.ok( + launcherSource.includes('--disable-blink-features=AutomationControlled'), + 'should disable AutomationControlled' + ); + }); +}); diff --git a/tests/web-ctl-actions.test.js b/tests/web-ctl-actions.test.js index 053c649..3d1ac23 100644 --- a/tests/web-ctl-actions.test.js +++ b/tests/web-ctl-actions.test.js @@ -837,6 +837,54 @@ describe('content blocking detection in goto', () => { }); }); +describe('auto headed fallback in goto', () => { + const fs = require('fs'); + const path = require('path'); + const webCtlSource = fs.readFileSync( + path.join(__dirname, '..', 'scripts', 'web-ctl.js'), + 'utf8' + ); + + it('--no-auto-recover is a valid boolean flag', () => { + assert.ok( + webCtlSource.includes("'--no-auto-recover'"), + '--no-auto-recover should be in BOOLEAN_FLAGS' + ); + }); + + it('goto case checks noAutoRecover flag', () => { + assert.ok( + webCtlSource.includes('noAutoRecover'), + 'goto case should check noAutoRecover flag' + ); + }); + + it('goto case includes headedFallback in result', () => { + assert.ok( + webCtlSource.includes('headedFallback: true'), + 'result should include headedFallback: true when fallback used' + ); + assert.ok( + webCtlSource.includes('headedFallback: false'), + 'result should include headedFallback: false when fallback unavailable' + ); + }); + + it('goto case launches headed browser on content block', () => { + assert.ok( + webCtlSource.includes('Content blocked in headless - falling back to headed browser'), + 'should warn about headed fallback' + ); + }); + + it('goto case handles headed fallback errors gracefully', () => { + assert.ok( + webCtlSource.includes('Headed fallback failed'), + 'should handle headed fallback errors' + ); + }); +}); + describe('--ensure-auth flag', () => { const fs = require('fs'); const path = require('path'); From 3f5d475b871b4af3ab6aa810a6acd1b07abf7d11 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 26 Feb 2026 04:06:53 +0200 Subject: [PATCH 4/5] docs: document stealth hardening and auto headed fallback --- CHANGELOG.md | 2 ++ README.md | 3 ++- skills/web-browse/SKILL.md | 8 +++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbaaaba..4253669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ - `--max-field-length ` flag for `extract` macro to configure maximum characters per extracted field (default: 500, max: 2000) - `--wait-loaded` flag for goto action - waits for async-rendered content to finish loading before taking the snapshot. Combines network idle, DOM stability, and loading indicator absence detection (spinners, skeletons, progress bars, aria-busy). Use `--timeout ` to set wait timeout (default: 15000ms) - Automatic content blocking detection in goto action - detects when sites serve pages but block content from headless browsers (e.g., X.com empty timelines). Uses provider-specific heuristics (content selectors, blocked indicators) and generic checks (empty content, persistent spinners). Response includes `contentBlocked: true`, `warning: 'content_blocked'`, and recovery suggestions. Disable with `--no-content-block-detect` flag +- Deep stealth hardening for headless browsers - CDP artifact removal, screen/viewport dimension spoofing, navigator.connection, WebRTC IP leak prevention. Reduces detection by aggressive anti-bot sites +- Auto headed fallback when content is blocked in headless - automatically switches to a headed browser to retrieve content when headless is detected and blocked. Response includes `headedFallback: true`. Disable with `--no-auto-recover` flag ### Fixed - Smart default snapshot scoping now includes complementary ARIA landmarks (`