test(docs): ✅ update test to use httpx instead of grequests

Author: Gabriella Martin

tests/test_docs.py

@@ -1,13 +1,16 @@
-from bs4 import BeautifulSoup
-import grequests
-import markdown
+import asyncio
 import os
 import unittest
+
+import httpx
+import markdown
+from bs4 import BeautifulSoup
+
 from . import (
+    DOC_NAMES,
+    DOCS_DIR,
     SCHEMA_NAMES,
     all_properties,
-    DOCS_DIR,
-    DOC_NAMES,
     property_doc_name,
     schema_enum_registry,
 )
@@ -117,44 +120,45 @@ def error_msg(schema_name, value, enum):
                             schema_name, v, enum
                         )  # noqa
 
-    def test_urls_in_docs(s):
-        def exception(request, exception):
-            return f"{request} - {exception}"
+    def test_urls_in_docs(self):
+        async def async_requests(urls):
+            async with httpx.AsyncClient(timeout=60) as client:
+                responses = (client.get(url) for url in urls)
+                results = await asyncio.gather(*responses, return_exceptions=True)
 
-        def async_requests(urls):
-            results = grequests.map(
-                (grequests.get(u) for u in urls), exception_handler=exception, size=100
-            )
             return results
 
-        urls = []
+        urls = ["https://www.google.com/", "https://www.google.com/404", "sdfghjk"]
 
-        for docname in DOC_NAMES:
-            filename = os.path.join(DOCS_DIR, f"{docname}.md")
-            with open(filename) as f:
-                doc_html = markdown.markdown(f.read())
-                soup = BeautifulSoup(doc_html, features="html.parser")
-                links = soup.find_all("a")
-                for link in links:
-                    url = link.get("href")
-                    if not url.startswith("http"):
-                        raise ValueError(f"Invalid URL in {docname}: {url}")
+        # for docname in DOC_NAMES:
+        #     filename = os.path.join(DOCS_DIR, f"{docname}.md")
+        #     with open(filename) as f:
+        #         doc_html = markdown.markdown(f.read())
+        #         soup = BeautifulSoup(doc_html, features="html.parser")
+        #         links = soup.find_all("a")
+        #         for link in links:
+        #             url = link.get("href")
+        #             if not url.startswith("http"):
+        #                 raise ValueError(f"Invalid URL in {docname}: {url}")
 
-                    urls.append(url)
+        #             urls.append(url)
 
-        results = async_requests(urls)
+        results = asyncio.run(async_requests(urls))
 
         warns = []
         not_founds = []
-        for resp in results:
-            if not resp.ok:
-                warns.append(f"failed {resp.status_code}: {resp.url}")
-            if resp.status_code in [404]:
-                not_founds.append(resp.url)
+        for response in results:
+            if isinstance(response, httpx.HTTPError):
+                warns.append(f"failed {response!s}: {response.request.url!s}")
+            else:
+                if not response.is_success:
+                    warns.append(f"failed {response.status_code}: {response.url!s}")
 
-        if not_founds:
-            raise ValueError(f"URLs not found: \n {not_founds}")
+                if response.status_code in (404,):
+                    not_founds.append(str(response.url))
 
         print("\n=== Minor URL link warnings ===\n")
         for w in warns:
             print(w)
+
+        assert not not_founds, f"URLs not found: \n {not_founds}"