From 16e45c497c115e7bf26dd1649f66937623212fec Mon Sep 17 00:00:00 2001
From: lululombard <lululombard@gmail.com>
Date: Sat, 6 Apr 2024 17:04:47 +0000
Subject: [PATCH] Fix logging and add retries
---
download/manager.py | 67 +++++++++++++++++++++----------------
redcraft_plugins_updater.py | 3 ++
requirements.txt | 2 +-
3 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/download/manager.py b/download/manager.py
index d8b4069..3222238 100644
--- a/download/manager.py
+++ b/download/manager.py
@@ -2,6 +2,7 @@
import traceback
import os
import asyncio
+import time
from download import sources
from download import post_processors
@@ -61,35 +62,43 @@ def __getitem__(self, item):
)
@classmethod
- async def download(self, source, name, url, post_processors, **kwargs):
- try:
- # Download file from the right source
- source = self.get_source_manager(source)
- downloaded_binary = await source.download_element(url, **kwargs)
-
- if not downloaded_binary:
- raise ValueError(
- "Downloaded empty binary for {} from {}".format(name, url)
- )
-
- # Run post_processors
- for post_processor in post_processors:
- processor = self.get_postprocessing_manager(post_processor)
- downloaded_binary, source, name, url = processor.process(
- downloaded_binary, source, name, url, **kwargs
- )
-
- # Save plugin somewhere
- destination = self.get_destination_manager()
- destination.save(downloaded_binary, source, name, url, **kwargs)
- self.logger.info("Downloaded {} from {}".format(name, url))
- except Exception:
- self.logger.error(
- "Error downloading {} from {} with post_processors {}".format(
- name, url, post_processors
- )
- )
- self.logger.error(traceback.format_exc())
+ async def download(self, source, name, url, post_processors, max_tries=5, **kwargs):
+ tries = 0
+ while True:
+ try:
+ # Download file from the right source
+ source_manager = self.get_source_manager(source)
+ downloaded_binary = await source_manager.download_element(url, **kwargs)
+
+ if not downloaded_binary:
+ raise ValueError(
+ "Downloaded empty binary for {} from {}".format(name, url)
+ )
+
+ # Run post_processors
+ for post_processor in post_processors:
+ processor = self.get_postprocessing_manager(post_processor)
+ downloaded_binary, source_manager, name, url = processor.process(
+ downloaded_binary, source_manager, name, url, **kwargs
+ )
+
+ # Save plugin somewhere
+ destination = self.get_destination_manager()
+ destination.save(downloaded_binary, source_manager, name, url, **kwargs)
+ self.logger.info("Downloaded {} from {}".format(name, url))
+ break
+ except Exception:
+ tries += 1
+ if tries >= max_tries:
+ self.logger.error(
+ "Error downloading {} from {} with post_processors {}".format(
+ name, url, post_processors
+ )
+ )
+ self.logger.error(traceback.format_exc())
+ break
+ else:
+ time.sleep(5 * tries)
@classmethod
async def download_resources(self, resources):
diff --git a/redcraft_plugins_updater.py b/redcraft_plugins_updater.py
index 1f2542e..cb6556c 100644
--- a/redcraft_plugins_updater.py
+++ b/redcraft_plugins_updater.py
@@ -1,5 +1,6 @@
import json
import argparse
+import logging
from dotenv import load_dotenv
import asyncio
@@ -10,6 +11,8 @@
if __name__ == "__main__":
load_dotenv()
+ logging.basicConfig(level=logging.INFO)
+
parser = argparse.ArgumentParser(description="Update plugins")
parser.add_argument(
"plugin_json_file",
diff --git a/requirements.txt b/requirements.txt
index c06b307..e8c3dde 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,6 @@ beautifulsoup4==4.12.2
PyYAML==6.0
boto3==1.26.133
cloudscraper >= 1.2, <= 1.3 # cloudscraper is often updated to bypass new Cloudflare challenges
-httpx==0.24.0
+httpx==0.27.0
jproperties==2.1.1
pyotp==2.9.0