From f3eb77d3286741ff8a03b222294d428ac371446e Mon Sep 17 00:00:00 2001
From: "copilot-developer-agent-robch[bot]"
 <175728472+Copilot@users.noreply.github.com>
Date: Fri, 21 Feb 2025 04:45:52 +0000
Subject: [PATCH 1/2] Initial plan for issue


From bfbd01f9985ff5525af94bdad2b8e2f9024e9d10 Mon Sep 17 00:00:00 2001
From: "copilot-developer-agent-robch[bot]"
 <175728472+Copilot@users.noreply.github.com>
Date: Fri, 21 Feb 2025 04:49:11 +0000
Subject: [PATCH 2/2] Add --wait-for UI element capabilities to mdx web

---
 src/Commands/CommandLineOptions.cs   | 10 ++++++++++
 src/Commands/WebCommand.cs           |  3 +++
 src/Helpers/PlaywrightHelpers.cs     | 18 +++++++++++++++++-
 src/assets/help/web get examples.txt |  4 ++++
 src/assets/help/web get options.txt  |  1 +
 5 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/Commands/CommandLineOptions.cs b/src/Commands/CommandLineOptions.cs
index 87536bd..f261c30 100644
--- a/src/Commands/CommandLineOptions.cs
+++ b/src/Commands/CommandLineOptions.cs
@@ -497,6 +497,16 @@ private static bool TryParseWebCommandOptions(WebCommand command, string[] args,
         {
             command.SearchProvider = WebSearchProvider.Bing;
         }
+        else if (arg == "--wait-for")
+        {
+            var selectors = GetInputOptionArgs(i + 1, args);
+            if (selectors.Count() == 0)
+            {
+                throw new CommandLineException($"Missing CSS selectors for {arg}");
+            }
+            command.WaitForSelectors.AddRange(selectors);
+            i += selectors.Count();
+        }
         else if (arg == "--duck-duck-go" || arg == "--duckduckgo")
         {
             command.SearchProvider = WebSearchProvider.DuckDuckGo;
diff --git a/src/Commands/WebCommand.cs b/src/Commands/WebCommand.cs
index c135131..40a739b 100644
--- a/src/Commands/WebCommand.cs
+++ b/src/Commands/WebCommand.cs
@@ -37,4 +37,7 @@ public WebCommand()
     public List<Tuple<string, string>> PageInstructionsList;
 
     public string SavePageOutput { get; set; }
+
+    // List of CSS selectors to wait for before proceeding
+    public List<string> WaitForSelectors { get; set; } = new();
 }
diff --git a/src/Helpers/PlaywrightHelpers.cs b/src/Helpers/PlaywrightHelpers.cs
index 35a3870..1245746 100644
--- a/src/Helpers/PlaywrightHelpers.cs
+++ b/src/Helpers/PlaywrightHelpers.cs
@@ -48,7 +48,7 @@ public static async Task<List<string>> GetWebSearchResultUrlsAsync(string search
         return urls;
     }
 
-    public static async Task<(string, string)> GetPageAndTitle(string url, bool stripHtml, string saveToFolder, BrowserType browserType, bool interactive)
+    public static async Task<(string, string)> GetPageAndTitle(string url, bool stripHtml, string saveToFolder, BrowserType browserType, bool interactive, List<string> waitForSelectors = null)
     {
         // Initialize Playwright
         using var playwright = await Playwright.CreateAsync();
@@ -59,6 +59,22 @@ public static async Task<List<string>> GetWebSearchResultUrlsAsync(string search
         // Navigate to the URL
         await page.GotoAsync(url);
 
+        // Wait for selectors if specified 
+        if (waitForSelectors?.Any() == true)
+        {
+            foreach (var selector in waitForSelectors)
+            {
+                try
+                {
+                    await page.WaitForSelectorAsync(selector, new() { Timeout = 30000 });
+                }
+                catch (TimeoutException)
+                {
+                    throw new Exception($"Timeout waiting for selector '{selector}' after 30000ms");
+                }
+            }
+        }
+
         // Fetch the page content and title
         var content = await FetchPageContent(page, url, stripHtml, saveToFolder);
         var title = await page.TitleAsync();
diff --git a/src/assets/help/web get examples.txt b/src/assets/help/web get examples.txt
index 50bd23d..f118cf5 100644
--- a/src/assets/help/web get examples.txt	
+++ b/src/assets/help/web get examples.txt	
@@ -23,6 +23,10 @@ EXAMPLES
 
     mdx web get https://example.com https://mbers.us/bio --instructions "style example.com as the other site"    
 
+  EXAMPLE 5: Wait for dynamic content to load
+
+    mdx web get https://example.com --wait-for "#main-content" ".dynamic-widget"
+
 SEE ALSO
 
   mdx help web get
diff --git a/src/assets/help/web get options.txt b/src/assets/help/web get options.txt
index 92d0a7d..cc50c48 100644
--- a/src/assets/help/web get options.txt	
+++ b/src/assets/help/web get options.txt	
@@ -13,6 +13,7 @@ OPTIONS
     --firefox                          Use Firefox browser
     --webkit                           Use WebKit browser
     --strip                            Strip HTML tags from downloaded content (default: false)
+    --wait-for SELECTOR [...]          Wait for specified CSS selectors to be present before proceeding
 
   AI PROCESSING