Add --wait-for UI element capabilities to mdx web command

src/Commands/CommandLineOptions.cs

@@ -497,6 +497,16 @@ private static bool TryParseWebCommandOptions(WebCommand command, string[] args,
         {
             command.SearchProvider = WebSearchProvider.Bing;
         }
+        else if (arg == "--wait-for")
+        {
+            var selectors = GetInputOptionArgs(i + 1, args);
+            if (selectors.Count() == 0)
+            {
+                throw new CommandLineException($"Missing CSS selectors for {arg}");
+            }
+            command.WaitForSelectors.AddRange(selectors);
+            i += selectors.Count();
+        }
         else if (arg == "--duck-duck-go" || arg == "--duckduckgo")
         {
             command.SearchProvider = WebSearchProvider.DuckDuckGo;

src/Commands/WebCommand.cs

@@ -37,4 +37,7 @@ public WebCommand()
     public List<Tuple<string, string>> PageInstructionsList;
 
     public string SavePageOutput { get; set; }
+
+    // List of CSS selectors to wait for before proceeding
+    public List<string> WaitForSelectors { get; set; } = new();
 }

src/Helpers/PlaywrightHelpers.cs

@@ -48,7 +48,7 @@ public static async Task<List<string>> GetWebSearchResultUrlsAsync(string search
         return urls;
     }
 
-    public static async Task<(string, string)> GetPageAndTitle(string url, bool stripHtml, string saveToFolder, BrowserType browserType, bool interactive)
+    public static async Task<(string, string)> GetPageAndTitle(string url, bool stripHtml, string saveToFolder, BrowserType browserType, bool interactive, List<string> waitForSelectors = null)
     {
         // Initialize Playwright
         using var playwright = await Playwright.CreateAsync();
@@ -59,6 +59,22 @@ public static async Task<List<string>> GetWebSearchResultUrlsAsync(string search
         // Navigate to the URL
         await page.GotoAsync(url);
 
+        // Wait for selectors if specified 
+        if (waitForSelectors?.Any() == true)
+        {
+            foreach (var selector in waitForSelectors)
+            {
+                try
+                {
+                    await page.WaitForSelectorAsync(selector, new() { Timeout = 30000 });
+                }
+                catch (TimeoutException)
+                {
+                    throw new Exception($"Timeout waiting for selector '{selector}' after 30000ms");
+                }
+            }
+        }
+
         // Fetch the page content and title
         var content = await FetchPageContent(page, url, stripHtml, saveToFolder);
         var title = await page.TitleAsync();

src/assets/help/web get examples.txt

@@ -23,6 +23,10 @@ EXAMPLES
 
     mdx web get https://example.com https://mbers.us/bio --instructions "style example.com as the other site"    
 
+  EXAMPLE 5: Wait for dynamic content to load
+
+    mdx web get https://example.com --wait-for "#main-content" ".dynamic-widget"
+
 SEE ALSO
 
   mdx help web get

src/assets/help/web get options.txt

@@ -13,6 +13,7 @@ OPTIONS
     --firefox                          Use Firefox browser
     --webkit                           Use WebKit browser
     --strip                            Strip HTML tags from downloaded content (default: false)
+    --wait-for SELECTOR [...]          Wait for specified CSS selectors to be present before proceeding
 
   AI PROCESSING