Fix retry arg and improve retry strategy (#362)

roboflow/__init__.py

@@ -15,7 +15,7 @@
 from roboflow.models import CLIPModel, GazeModel  # noqa: F401
 from roboflow.util.general import write_line
 
-__version__ = "1.1.54"
+__version__ = "1.1.55"
 
 
 def check_key(api_key, model, notebook, num_retries=0):

roboflow/adapters/rfapi.py

@@ -4,6 +4,7 @@
 from typing import Optional
 
 import requests
+from requests.exceptions import RequestException
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from roboflow.config import API_URL, DEFAULT_BATCH_NAME, DEFAULT_JOB_NAME
@@ -26,6 +27,7 @@ class AnnotationSaveError(RoboflowError):
     def __init__(self, message, status_code=None):
         self.message = message
         self.status_code = status_code
+        self.retries = 0
         super().__init__(self.message)
 
 
@@ -85,14 +87,21 @@ def upload_image(
                 "file": ("imageToUpload", imgjpeg, "image/jpeg"),
             }
         )
-        response = requests.post(upload_url, data=m, headers={"Content-Type": m.content_type}, timeout=(300, 300))
+
+        try:
+            response = requests.post(upload_url, data=m, headers={"Content-Type": m.content_type}, timeout=(300, 300))
+        except RequestException as e:
+            raise ImageUploadError(str(e)) from e
 
     else:
         # Hosted image upload url
         upload_url = _hosted_upload_url(api_key, project_url, image_path, split, coalesced_batch_name, tag_names)
 
-        # Get response
-        response = requests.post(upload_url, timeout=(300, 300))
+        try:
+            # Get response
+            response = requests.post(upload_url, timeout=(300, 300))
+        except RequestException as e:
+            raise ImageUploadError(str(e)) from e
 
     responsejson = None
     try:
@@ -147,12 +156,15 @@ def save_annotation(
         api_key, project_url, annotation_name, image_id, job_name, is_prediction, overwrite
     )
 
-    response = requests.post(
-        upload_url,
-        data=json.dumps({"annotationFile": annotation_string, "labelmap": annotation_labelmap}),
-        headers={"Content-Type": "application/json"},
-        timeout=(60, 60),
-    )
+    try:
+        response = requests.post(
+            upload_url,
+            data=json.dumps({"annotationFile": annotation_string, "labelmap": annotation_labelmap}),
+            headers={"Content-Type": "application/json"},
+            timeout=(60, 60),
+        )
+    except RequestException as e:
+        raise AnnotationSaveError(str(e)) from e
 
     # Handle response
     responsejson = None

roboflow/core/project.py

@@ -11,7 +11,7 @@
 import requests
 
 from roboflow.adapters import rfapi
-from roboflow.adapters.rfapi import ImageUploadError
+from roboflow.adapters.rfapi import AnnotationSaveError, ImageUploadError
 from roboflow.config import API_URL, DEMO_KEYS
 from roboflow.core.version import Version
 from roboflow.util.general import Retry
@@ -515,26 +515,34 @@ def save_annotation(
         job_name=None,
         is_prediction: bool = False,
         annotation_overwrite=False,
+        num_retry_uploads=0,
     ):
         project_url = self.id.rsplit("/")[1]
         annotation_name, annotation_str = self._annotation_params(annotation_path)
         t0 = time.time()
+        upload_retry_attempts = 0
+        retry = Retry(num_retry_uploads, AnnotationSaveError)
 
-        annotation = rfapi.save_annotation(
-            self.__api_key,
-            project_url,
-            annotation_name,  # type: ignore[type-var]
-            annotation_str,  # type: ignore[type-var]
-            image_id,
-            job_name=job_name,  # type: ignore[type-var]
-            is_prediction=is_prediction,
-            annotation_labelmap=annotation_labelmap,
-            overwrite=annotation_overwrite,
-        )
+        try:
+            annotation = rfapi.save_annotation(
+                self.__api_key,
+                project_url,
+                annotation_name,  # type: ignore[type-var]
+                annotation_str,  # type: ignore[type-var]
+                image_id,
+                job_name=job_name,  # type: ignore[type-var]
+                is_prediction=is_prediction,
+                annotation_labelmap=annotation_labelmap,
+                overwrite=annotation_overwrite,
+            )
+            upload_retry_attempts = retry.retries
+        except AnnotationSaveError as e:
+            e.retries = upload_retry_attempts
+            raise
 
         upload_time = time.time() - t0
 
-        return annotation, upload_time
+        return annotation, upload_time, upload_retry_attempts
 
     def single_upload(
         self,
@@ -563,6 +571,7 @@ def single_upload(
         uploaded_image, uploaded_annotation = None, None
         upload_time, annotation_time = None, None
         upload_retry_attempts = 0
+        annotation_upload_retry_attempts = 0
 
         if image_path:
             uploaded_image, upload_time, upload_retry_attempts = self.upload_image(
@@ -579,13 +588,14 @@ def single_upload(
             image_id = uploaded_image["id"]  # type: ignore[index]
 
         if annotation_path and image_id:
-            uploaded_annotation, annotation_time = self.save_annotation(
+            uploaded_annotation, annotation_time, annotation_upload_retry_attempts = self.save_annotation(
                 annotation_path,
                 annotation_labelmap,
                 image_id,
                 batch_name,
                 is_prediction,
                 annotation_overwrite,
+                num_retry_uploads=num_retry_uploads,
             )
 
         return {
@@ -594,6 +604,7 @@ def single_upload(
             "upload_time": upload_time,
             "annotation_time": annotation_time,
             "upload_retry_attempts": upload_retry_attempts,
+            "annotation_upload_retry_attempts": annotation_upload_retry_attempts,
         }
 
     def _annotation_params(self, annotation_path):

roboflow/core/workspace.py

@@ -349,6 +349,7 @@ def _upload_image(imagedesc):
                 batch_name=batch_name,
                 sequence_number=imagedesc.get("index"),
                 sequence_size=len(images),
+                num_retry_uploads=num_retries,
             )
 
             return image, upload_time, upload_retry_attempts
@@ -376,6 +377,7 @@ def _save_annotation(image_id, imagedesc):
                 annotation_labelmap=labelmap,
                 image_id=image_id,
                 job_name=batch_name,
+                num_retry_uploads=num_retries,
             )
 
             return annotation, upload_time

roboflow/util/general.py

@@ -1,4 +1,6 @@
 import sys
+import time
+from random import random
 
 
 def write_line(line):
@@ -13,8 +15,16 @@ def __init__(self, max_retries, retry_on):
         self.retry_on = retry_on
         self.retries = 0
 
+    def backoff(self):
+        """
+        Backoff for a random time based on number of retries.
+        """
+        base_t_ms = 100
+        max_t_ms = 30000
+        sleep_ms = random() * min(max_t_ms, base_t_ms * 2**self.retries)
+        time.sleep(int(sleep_ms) / 1000)
+
     def __call__(self, func, *args, **kwargs):
-        self.retries = 0
         retry_on = self.retry_on
         if not retry_on:
             retry_on = (Exception,)
@@ -24,8 +34,9 @@ def __call__(self, func, *args, **kwargs):
                 return func(*args, **kwargs)
             except BaseException as e:
                 if isinstance(e, retry_on):
-                    self.retries += 1
-                    if self.retries > self.max_retries:
+                    if self.retries >= self.max_retries:
                         raise
+                    self.backoff()
+                    self.retries += 1
                 else:
                     raise