feat(goto): add --wait-loaded flag for async-rendered content

CHANGELOG.md

@@ -26,6 +26,7 @@
 - `verifyUrl` and `verifySelector` provider fields for built-in providers (github, gitlab, microsoft) to automatically verify API/dashboard access after login
 - `--min-wait <seconds>` flag for `session auth` to configure grace period before auth success polling starts (default: 5 seconds, clamped to 0-300)
 - `--max-field-length <N>` flag for `extract` macro to configure maximum characters per extracted field (default: 500, max: 2000)
+- `--wait-loaded` flag for goto action - waits for async-rendered content to finish loading before taking the snapshot. Combines network idle, DOM stability, and loading indicator absence detection (spinners, skeletons, progress bars, aria-busy). Use `--timeout <ms>` to set wait timeout (default: 15000ms)
 
 ### Fixed
 - Smart default snapshot scoping now includes complementary ARIA landmarks (`<aside>`, `[role="complementary"]`) alongside `<main>`, capturing sidebar content like repository stats (#26)

README.md

@@ -94,7 +94,7 @@ web-ctl session end github
 
 | Action | Usage | Returns |
 |--------|-------|---------|
-| `goto` | `run <s> goto <url> [--no-auth-wall-detect] [--ensure-auth]` | `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, snapshot }` |
+| `goto` | `run <s> goto <url> [--no-auth-wall-detect] [--ensure-auth] [--wait-loaded]` | `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, waitLoaded, snapshot }` |
 | `snapshot` | `run <s> snapshot` | `{ url, snapshot }` |
 | `click` | `run <s> click <sel> [--wait-stable]` | `{ url, clicked, snapshot }` |
 | `click-wait` | `run <s> click-wait <sel> [--timeout]` | `{ url, clicked, settled, snapshot }` |
@@ -174,6 +174,7 @@ This eliminates the common click-snapshot-check loop that wastes agent turns on
 | `--allow-evaluate` | `evaluate` | Required safety flag for JS execution |
 | `--no-auth-wall-detect` | `goto` | Disable automatic auth wall detection and checkpoint opening |
 | `--ensure-auth` | `goto` | Poll for auth completion instead of timed checkpoint; overrides `--no-auth-wall-detect` |
+| `--wait-loaded` | `goto` | Wait for async content to finish rendering (network idle + loading indicator absence + DOM quiet) |
 | `--snapshot-depth <N>` | Any action with snapshot | Limit ARIA tree depth (e.g. 3 for top 3 levels) |
 | `--snapshot-selector <sel>` | Any action with snapshot | Scope snapshot to a DOM subtree |
 | `--snapshot-max-lines <N>` | Any action with snapshot | Truncate snapshot to N lines |

scripts/browser-launcher.js

@@ -195,4 +195,65 @@ async function waitForStable(page, { timeout = 5000 } = {}) {
   }), DOM_QUIET_MS);
 }
 
-module.exports = { launchBrowser, closeBrowser, randomDelay, isWSL, canLaunchHeaded, waitForStable };
+/**
+ * Wait for async-rendered content to finish loading.
+ * Combines network idle, DOM stability, and loading indicator absence detection.
+ *
+ * @param {import('playwright').Page} page
+ * @param {object} options - { timeout: number (ms, default 15000) }
+ */
+async function waitForLoaded(page, { timeout = 15000 } = {}) {
+  await waitForStable(page, { timeout });
+
+  const deadline = Date.now() + timeout;
+  const POLL_MS = 200;
+  await page.evaluate(({ pollMs, deadlineTs }) => new Promise(resolve => {
+    const SELECTORS = [
+      '[role="progressbar"]', '[aria-busy="true"]',
+      '.loading', '.spinner', '.skeleton',
+      '[class*="loading"]', '[class*="spinner"]', '[class*="skeleton"]'
+    ];
+    const TEXT_RE = /^\s*(loading|please wait|crunching)\.*\s*$/i;
+
+    function hasLoadingIndicators() {
+      if (!document.body) return false;
+      const match = document.body.querySelectorAll(SELECTORS.join(','));
+      for (const el of match) {
+        if (el.offsetParent !== null) return true;
+      }
+      const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+      let nodeCount = 0;
+      while (walker.nextNode() && nodeCount++ < 5000) {
+        if (TEXT_RE.test(walker.currentNode.textContent)) return true;
+      }
+      return false;
+    }
+
+    function poll() {
+      if (!hasLoadingIndicators() || Date.now() >= deadlineTs) {
+        resolve();
+      } else {
+        setTimeout(poll, pollMs);
+      }
+    }
+    poll();
+  }), { pollMs: POLL_MS, deadlineTs: deadline });
+
+  const remaining = Math.max(deadline - Date.now(), 0);
+  if (remaining > 300) {
+    const DOM_QUIET_MS = 300;
+    await page.evaluate((ms) => new Promise(resolve => {
+      if (!document.body) { resolve(); return; }
+      const done = () => { observer.disconnect(); resolve(); };
+      let timer = setTimeout(done, ms);
+      const observer = new MutationObserver(() => {
+        clearTimeout(timer);
+        timer = setTimeout(done, ms);
+      });
+      observer.observe(document.body, { childList: true, subtree: true, attributes: true });
+      setTimeout(done, ms * 5);
+    }), DOM_QUIET_MS);
+  }
+}
+
+module.exports = { launchBrowser, closeBrowser, randomDelay, isWSL, canLaunchHeaded, waitForStable, waitForLoaded };

scripts/web-ctl.js

@@ -2,7 +2,7 @@
 'use strict';
 
 const sessionStore = require('./session-store');
-const { launchBrowser, closeBrowser, randomDelay, waitForStable, canLaunchHeaded } = require('./browser-launcher');
+const { launchBrowser, closeBrowser, randomDelay, waitForStable, waitForLoaded, canLaunchHeaded } = require('./browser-launcher');
 const { detectAuthWall } = require('./auth-wall-detect');
 const { runAuthFlow } = require('./auth-flow');
 const { checkAuthSuccess } = require('./auth-check');
@@ -21,7 +21,7 @@ const BOOLEAN_FLAGS = new Set([
   '--allow-evaluate', '--no-snapshot', '--wait-stable', '--vnc',
   '--exact', '--accept', '--submit', '--dismiss', '--auto',
   '--snapshot-collapse', '--snapshot-text-only', '--snapshot-compact',
-  '--snapshot-full', '--no-auth-wall-detect', '--ensure-auth',
+  '--snapshot-full', '--no-auth-wall-detect', '--ensure-auth', '--wait-loaded',
 ]);
 
 function validateSessionName(name) {
@@ -950,6 +950,7 @@ async function runAction(sessionName, action, actionArgs, opts) {
         const url = actionArgs[0];
         if (!url) throw new Error('URL required: run <session> goto <url>');
         validateUrl(url);
+        const loadedTimeout = opts.timeout ? parseInt(opts.timeout, 10) : 15000;
         const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
         if (opts.ensureAuth || !opts.noAuthWallDetect) {
           const detection = await detectAuthWall(page, context, url);
@@ -986,8 +987,12 @@ async function runAction(sessionName, action, actionArgs, opts) {
                     context = headlessBrowser.context;
                     page = headlessBrowser.page;
                     await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+                    if (opts.waitLoaded) {
+                      await waitForLoaded(page, { timeout: loadedTimeout });
+                    }
                     const snapshot = await getSnapshot(page, opts);
                     result = { url: page.url(), authWallDetected: true, ensureAuthCompleted: true,
+                               ...(opts.waitLoaded && { waitLoaded: true }),
                                ...(snapshot != null && { snapshot }) };
                   } catch (relaunchErr) {
                     result = { url, authWallDetected: true, ensureAuthCompleted: true,
@@ -1007,8 +1012,12 @@ async function runAction(sessionName, action, actionArgs, opts) {
               } else {
                 console.warn('[WARN] Checkpoint open for ' + (ckTimeout / 1000) + 's');
                 await new Promise(resolve => setTimeout(resolve, ckTimeout));
+                if (opts.waitLoaded) {
+                  await waitForLoaded(page, { timeout: loadedTimeout });
+                }
                 const snapshot = await getSnapshot(page, opts);
                 result = { url: page.url(), authWallDetected: true, checkpointCompleted: true,
+                           ...(opts.waitLoaded && { waitLoaded: true }),
                            ...(snapshot != null && { snapshot }) };
                 break;
               }
@@ -1018,16 +1027,23 @@ async function runAction(sessionName, action, actionArgs, opts) {
                            message: 'Auth wall detected but no display available for headed browser.' };
                 break;
               }
+              if (opts.waitLoaded) {
+                await waitForLoaded(page, { timeout: loadedTimeout });
+              }
               const snapshot = await getSnapshot(page, opts);
               result = { url: page.url(), authWallDetected: true, checkpointCompleted: false,
                          message: 'Auth wall detected but no display for headed checkpoint.',
+                         ...(opts.waitLoaded && { waitLoaded: true }),
                          ...(snapshot != null && { snapshot }) };
               break;
             }
           }
         }
+        if (opts.waitLoaded) {
+          await waitForLoaded(page, { timeout: loadedTimeout });
+        }
         const snapshot = await getSnapshot(page, opts);
-        result = { url: page.url(), status: response ? response.status() : null, ...(snapshot != null && { snapshot }) };
+        result = { url: page.url(), status: response ? response.status() : null, ...(opts.waitLoaded && { waitLoaded: true }), ...(snapshot != null && { snapshot }) };
         break;
       }
 
@@ -1167,7 +1183,7 @@ async function runAction(sessionName, action, actionArgs, opts) {
       default: {
         const macro = macros[action];
         if (macro) {
-          const helpers = { resolveSelector, waitForStable, randomDelay, getSnapshot: (page) => getSnapshot(page, opts), sanitizeWebContent };
+          const helpers = { resolveSelector, waitForStable, waitForLoaded, randomDelay, getSnapshot: (page) => getSnapshot(page, opts), sanitizeWebContent };
           result = await macro(page, actionArgs, opts, helpers);
           // Clean up null snapshot from macros when --no-snapshot is active
           if (result && result.snapshot == null) delete result.snapshot;
@@ -1251,6 +1267,8 @@ Session commands:
 Run actions:
   goto <url>                    Navigate to URL
     [--ensure-auth]             Poll for auth completion instead of timed checkpoint
+    [--wait-loaded]             Wait for async content to finish rendering
+    [--timeout <ms>]            Wait timeout (default: 15000)
   snapshot                      Get accessibility tree
   click <selector>              Click element
     [--wait-stable]             Wait for DOM + network to settle after click

skills/web-browse/SKILL.md

@@ -46,7 +46,7 @@ Safe practice: always double-quote URL arguments.
 ### goto - Navigate to URL
 
 ```bash
-node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url> [--no-auth-wall-detect] [--ensure-auth]
+node ${PLUGIN_ROOT}/scripts/web-ctl.js run <session> goto <url> [--no-auth-wall-detect] [--ensure-auth] [--wait-loaded]
 ```
 
 Navigates to a URL and automatically detects authentication walls using a three-heuristic detection system:
@@ -60,7 +60,9 @@ Use `--no-auth-wall-detect` to disable this automatic detection and skip the che
 
 Use `--ensure-auth` to actively poll for authentication completion instead of a timed checkpoint. When set, the headed browser polls with `checkAuthSuccess` at 2-second intervals using the URL-change heuristic. On success, the headed browser closes, a headless browser relaunches, and the original URL is loaded. On timeout, returns `ensureAuthCompleted: false`. This flag overrides `--no-auth-wall-detect`.
 
-Returns: `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, snapshot }`
+Use `--wait-loaded` to wait for async-rendered content to finish loading before taking the snapshot. This combines network idle, DOM stability, loading indicator absence detection (spinners, skeletons, progress bars, aria-busy), and a final DOM quiet period. Use `--timeout <ms>` to set the wait timeout (default: 15000ms). Ideal for SPAs and pages that render content after the initial page load.
+
+Returns: `{ url, status, authWallDetected, checkpointCompleted, ensureAuthCompleted, waitLoaded, snapshot }`
 
 ### snapshot - Get Accessibility Tree