Fix logging and add retries

download/manager.py

@@ -2,6 +2,7 @@
 import traceback
 import os
 import asyncio
+import time
 
 from download import sources
 from download import post_processors
@@ -61,35 +62,43 @@ def __getitem__(self, item):
     )
 
     @classmethod
-    async def download(self, source, name, url, post_processors, **kwargs):
-        try:
-            # Download file from the right source
-            source = self.get_source_manager(source)
-            downloaded_binary = await source.download_element(url, **kwargs)
-
-            if not downloaded_binary:
-                raise ValueError(
-                    "Downloaded empty binary for {} from {}".format(name, url)
-                )
-
-            # Run post_processors
-            for post_processor in post_processors:
-                processor = self.get_postprocessing_manager(post_processor)
-                downloaded_binary, source, name, url = processor.process(
-                    downloaded_binary, source, name, url, **kwargs
-                )
-
-            # Save plugin somewhere
-            destination = self.get_destination_manager()
-            destination.save(downloaded_binary, source, name, url, **kwargs)
-            self.logger.info("Downloaded {} from {}".format(name, url))
-        except Exception:
-            self.logger.error(
-                "Error downloading {} from {} with post_processors {}".format(
-                    name, url, post_processors
-                )
-            )
-            self.logger.error(traceback.format_exc())
+    async def download(self, source, name, url, post_processors, max_tries=5, **kwargs):
+        tries = 0
+        while True:
+            try:
+                # Download file from the right source
+                source_manager = self.get_source_manager(source)
+                downloaded_binary = await source_manager.download_element(url, **kwargs)
+
+                if not downloaded_binary:
+                    raise ValueError(
+                        "Downloaded empty binary for {} from {}".format(name, url)
+                    )
+
+                # Run post_processors
+                for post_processor in post_processors:
+                    processor = self.get_postprocessing_manager(post_processor)
+                    downloaded_binary, source_manager, name, url = processor.process(
+                        downloaded_binary, source_manager, name, url, **kwargs
+                    )
+
+                # Save plugin somewhere
+                destination = self.get_destination_manager()
+                destination.save(downloaded_binary, source_manager, name, url, **kwargs)
+                self.logger.info("Downloaded {} from {}".format(name, url))
+                break
+            except Exception:
+                tries += 1
+                if tries >= max_tries:
+                    self.logger.error(
+                        "Error downloading {} from {} with post_processors {}".format(
+                            name, url, post_processors
+                        )
+                    )
+                    self.logger.error(traceback.format_exc())
+                    break
+                else:
+                    time.sleep(5 * tries)
 
     @classmethod
     async def download_resources(self, resources):

redcraft_plugins_updater.py

@@ -1,5 +1,6 @@
 import json
 import argparse
+import logging
 from dotenv import load_dotenv
 
 import asyncio
@@ -10,6 +11,8 @@
 if __name__ == "__main__":
     load_dotenv()
 
+    logging.basicConfig(level=logging.INFO)
+
     parser = argparse.ArgumentParser(description="Update plugins")
     parser.add_argument(
         "plugin_json_file",

requirements.txt

@@ -5,6 +5,6 @@ beautifulsoup4==4.12.2
 PyYAML==6.0
 boto3==1.26.133
 cloudscraper >= 1.2, <= 1.3 # cloudscraper is often updated to bypass new Cloudflare challenges
-httpx==0.24.0
+httpx==0.27.0
 jproperties==2.1.1
 pyotp==2.9.0