Merge pull request #320 from coderxio/jrlegrand/max-workers

Max workers 3

Author: jrlegrand

airflow/dags/sagerx.py

@@ -208,14 +208,16 @@ def get_api(url):
 def parallel_api_calls(api_calls:list) -> list:
     from concurrent.futures import ThreadPoolExecutor, as_completed
     output = []
-    with ThreadPoolExecutor(max_workers=32) as executor:
+    with ThreadPoolExecutor(max_workers=3) as executor:
         futures = {executor.submit(get_api, api_call):api_call for api_call in api_calls}
 
         for future in as_completed(futures):
             url = futures[future]
             response = future.result()
             if not len(response) == 0:
                 output.append({"url":url,"response":response})
-            else:
-                print(f"Empty response for url: {url}")
+                if len(output) % 1000 == 0:
+                    print(f'MILESTONE {len(output)}')
+            # else:
+            #     print(f"Empty response for url: {url}")
     return output