From 8a932392ffff4a0a187ade70cbd241ae760e2afe Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Mon, 11 Dec 2023 17:56:09 -0300 Subject: [PATCH] allow to configure retry logging --- shub_workflow/utils/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/shub_workflow/utils/__init__.py b/shub_workflow/utils/__init__.py index 7fd0d7f..e66bda5 100644 --- a/shub_workflow/utils/__init__.py +++ b/shub_workflow/utils/__init__.py @@ -61,13 +61,18 @@ def resolve_project_id(project_id=None) -> Optional[int]: ONE_MIN_IN_S = 60 +DASH_RETRY_MAX = int(os.environ.get("DASH_RETRY_MAX", MINS_IN_A_DAY)) +DASH_RETRY_WAIT_SECS = int(os.environ.get("DASH_RETRY_WAIT_SECS", ONE_MIN_IN_S)) +DASH_RETRY_LOGGING_LEVEL = os.environ.get("DASH_RETRY_LOGGING_LEVEL", "ERROR") + + dash_retry_decorator = retry( # ServerError is the only ScrapinghubAPIError that should be retried. Don't capture ScrapinghubAPIError here retry=retry_if_exception_type((ServerError, ReadTimeout, ConnectionError, HTTPError)), - before_sleep=before_sleep_log(logger, logging.ERROR), + before_sleep=before_sleep_log(logger, getattr(logging, DASH_RETRY_LOGGING_LEVEL)), reraise=True, - stop=stop_after_attempt(MINS_IN_A_DAY), - wait=wait_fixed(ONE_MIN_IN_S), + stop=stop_after_attempt(DASH_RETRY_MAX), + wait=wait_fixed(DASH_RETRY_WAIT_SECS), )