Don't shuffle proxies before checking

monosans · Jan 25, 2024 · 890c000 · 890c000
1 parent 3059940
commit 890c000
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 24 deletions.
diff --git a/proxy_scraper_checker/checker.py b/proxy_scraper_checker/checker.py
@@ -2,7 +2,6 @@
 
 import asyncio
 import logging
-from random import shuffle
 from typing import Mapping
 
 from aiohttp_socks import ProxyType
@@ -53,15 +52,15 @@ async def check_all(
         for proto in sort.PROTOCOL_ORDER
         if proto in storage.enabled_protocols
     }
-    coroutines = [
-        check_one(
-            progress=progress,
-            proxy=proxy,
-            settings=settings,
-            storage=storage,
-            task=tasks[proxy.protocol],
+    await asyncio.gather(
+        *(
+            check_one(
+                progress=progress,
+                proxy=proxy,
+                settings=settings,
+                storage=storage,
+                task=tasks[proxy.protocol],
+            )
+            for proxy in storage
         )
-        for proxy in storage
-    ]
-    shuffle(coroutines)
-    await asyncio.gather(*coroutines)
+    )
diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py
@@ -94,17 +94,18 @@ async def scrape_all(
         for proto, sources in settings.sources.items()
     }
     timeout = ClientTimeout(total=settings.source_timeout)
-    coroutines = (
-        scrape_one(
-            progress=progress,
-            proto=proto,
-            session=session,
-            source=source,
-            storage=storage,
-            task=tasks[proto],
-            timeout=timeout,
+    await asyncio.gather(
+        *(
+            scrape_one(
+                progress=progress,
+                proto=proto,
+                session=session,
+                source=source,
+                storage=storage,
+                task=tasks[proto],
+                timeout=timeout,
+            )
+            for proto, sources in settings.sources.items()
+            for source in sources
         )
-        for proto, sources in settings.sources.items()
-        for source in sources
     )
-    await asyncio.gather(*coroutines)