✨ Retry 5 times if ConnectionError

Gudsfile · Oct 12, 2024 · f467602 · f467602
1 parent 12b8812
commit f467602
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 5 deletions.
diff --git a/sploty/audio_features.py b/sploty/audio_features.py
@@ -6,7 +6,7 @@
 import pandas as pd
 import requests
 from pydantic import BaseModel, HttpUrl
-from requests.exceptions import HTTPError
+from requests.exceptions import ConnectionError, HTTPError
 from tinydb import TinyDB
 
 logger = logging.getLogger(__name__)
@@ -19,9 +19,10 @@ class SpotifyApiParams(BaseModel):
     headers: dict
     timeout: float
     sleep: float
+    connection_error_retry: int = 5
 
 
-def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None):
+def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None, retry=0):
     try:
         response = requests.get(
             spotify_api_params.url,
@@ -44,6 +45,18 @@ def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None):
             return do_spotify_request(spotify_api_params, params)
         logger.warning("HTTPError - %s (skipping)", err)
         raise
+    except ConnectionError as err:
+        if retry < spotify_api_params.connection_error_retry:
+            logger.warning(
+                "ConnectionError - %s (retry %i/5 sleeping %is...)",
+                err,
+                retry + 1,
+                spotify_api_params.sleep,
+            )
+            time.sleep(spotify_api_params.sleep * min(retry + 1))
+            return do_spotify_request(spotify_api_params, params, retry + 1)
+        logger.warning("ConnectionError - %s (skipping)", err)
+        raise
 
 
 def get_track_audio_features(spotify_api_params: SpotifyApiParams, track_uris):

diff --git a/sploty/enrich.py b/sploty/enrich.py
@@ -8,7 +8,7 @@
 import pandas as pd
 import requests
 from pydantic import BaseModel, HttpUrl
-from requests.exceptions import HTTPError
+from requests.exceptions import ConnectionError, HTTPError
 
 from sploty.settings import BoldColor
 
@@ -22,9 +22,10 @@ class SpotifyApiParams(BaseModel):
     headers: dict
     timeout: float
     sleep: float
+    connection_error_retry: int = 5
 
 
-def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None):
+def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None, retry=0):
     try:
         response = requests.get(
             spotify_api_params.url,
@@ -43,6 +44,18 @@ def do_spotify_request(spotify_api_params: SpotifyApiParams, params=None):
             return do_spotify_request(spotify_api_params, params)
         logger.warning("HTTPError - %s (skipping)", err)
         raise
+    except ConnectionError as err:
+        if retry < spotify_api_params.connection_error_retry:
+            logger.warning(
+                "ConnectionError - %s (retry %i/5 sleeping %is...)",
+                err,
+                retry + 1,
+                spotify_api_params.sleep,
+            )
+            time.sleep(spotify_api_params.sleep * min(retry + 1))
+            return do_spotify_request(spotify_api_params, params, retry + 1)
+        logger.warning("ConnectionError - %s (skipping)", err)
+        raise
 
 
 def another_get(spotify_api_params: SpotifyApiParams, track_uris):
@@ -139,7 +152,7 @@ def number_of_lines(file_path: str):
     fp = Path(file_path)
     if fp.exists():
         with fp.open(encoding="UTF-8") as file:
-            return sum(1 for _ in file)
+            return sum(1 for _ in file) - 1
     return 0