diff --git a/download/manager.py b/download/manager.py index d8b4069..3222238 100644 --- a/download/manager.py +++ b/download/manager.py @@ -2,6 +2,7 @@ import traceback import os import asyncio +import time from download import sources from download import post_processors @@ -61,35 +62,43 @@ def __getitem__(self, item): ) @classmethod - async def download(self, source, name, url, post_processors, **kwargs): - try: - # Download file from the right source - source = self.get_source_manager(source) - downloaded_binary = await source.download_element(url, **kwargs) - - if not downloaded_binary: - raise ValueError( - "Downloaded empty binary for {} from {}".format(name, url) - ) - - # Run post_processors - for post_processor in post_processors: - processor = self.get_postprocessing_manager(post_processor) - downloaded_binary, source, name, url = processor.process( - downloaded_binary, source, name, url, **kwargs - ) - - # Save plugin somewhere - destination = self.get_destination_manager() - destination.save(downloaded_binary, source, name, url, **kwargs) - self.logger.info("Downloaded {} from {}".format(name, url)) - except Exception: - self.logger.error( - "Error downloading {} from {} with post_processors {}".format( - name, url, post_processors - ) - ) - self.logger.error(traceback.format_exc()) + async def download(self, source, name, url, post_processors, max_tries=5, **kwargs): + tries = 0 + while True: + try: + # Download file from the right source + source_manager = self.get_source_manager(source) + downloaded_binary = await source_manager.download_element(url, **kwargs) + + if not downloaded_binary: + raise ValueError( + "Downloaded empty binary for {} from {}".format(name, url) + ) + + # Run post_processors + for post_processor in post_processors: + processor = self.get_postprocessing_manager(post_processor) + downloaded_binary, source_manager, name, url = processor.process( + downloaded_binary, source_manager, name, url, **kwargs + ) + + # Save plugin somewhere + destination = self.get_destination_manager() + destination.save(downloaded_binary, source_manager, name, url, **kwargs) + self.logger.info("Downloaded {} from {}".format(name, url)) + break + except Exception: + tries += 1 + if tries >= max_tries: + self.logger.error( + "Error downloading {} from {} with post_processors {}".format( + name, url, post_processors + ) + ) + self.logger.error(traceback.format_exc()) + break + else: + time.sleep(5 * tries) @classmethod async def download_resources(self, resources): diff --git a/redcraft_plugins_updater.py b/redcraft_plugins_updater.py index 1f2542e..cb6556c 100644 --- a/redcraft_plugins_updater.py +++ b/redcraft_plugins_updater.py @@ -1,5 +1,6 @@ import json import argparse +import logging from dotenv import load_dotenv import asyncio @@ -10,6 +11,8 @@ if __name__ == "__main__": load_dotenv() + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser(description="Update plugins") parser.add_argument( "plugin_json_file", diff --git a/requirements.txt b/requirements.txt index c06b307..e8c3dde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ beautifulsoup4==4.12.2 PyYAML==6.0 boto3==1.26.133 cloudscraper >= 1.2, <= 1.3 # cloudscraper is often updated to bypass new Cloudflare challenges -httpx==0.24.0 +httpx==0.27.0 jproperties==2.1.1 pyotp==2.9.0