From b9b479662051840728cf0a5f8d711d00b62678f4 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Mon, 10 Mar 2025 16:31:01 +0100 Subject: [PATCH] Fix retry arg and improve retry strategy (#362) --- roboflow/__init__.py | 2 +- roboflow/adapters/rfapi.py | 30 ++++++++++++++++++++--------- roboflow/core/project.py | 39 ++++++++++++++++++++++++-------------- roboflow/core/workspace.py | 2 ++ roboflow/util/general.py | 17 ++++++++++++++--- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/roboflow/__init__.py b/roboflow/__init__.py index 4b6d7476..851a5406 100644 --- a/roboflow/__init__.py +++ b/roboflow/__init__.py @@ -15,7 +15,7 @@ from roboflow.models import CLIPModel, GazeModel # noqa: F401 from roboflow.util.general import write_line -__version__ = "1.1.54" +__version__ = "1.1.55" def check_key(api_key, model, notebook, num_retries=0): diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index af5a24c8..0fd5a3ec 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -4,6 +4,7 @@ from typing import Optional import requests +from requests.exceptions import RequestException from requests_toolbelt.multipart.encoder import MultipartEncoder from roboflow.config import API_URL, DEFAULT_BATCH_NAME, DEFAULT_JOB_NAME @@ -26,6 +27,7 @@ class AnnotationSaveError(RoboflowError): def __init__(self, message, status_code=None): self.message = message self.status_code = status_code + self.retries = 0 super().__init__(self.message) @@ -85,14 +87,21 @@ def upload_image( "file": ("imageToUpload", imgjpeg, "image/jpeg"), } ) - response = requests.post(upload_url, data=m, headers={"Content-Type": m.content_type}, timeout=(300, 300)) + + try: + response = requests.post(upload_url, data=m, headers={"Content-Type": m.content_type}, timeout=(300, 300)) + except RequestException as e: + raise ImageUploadError(str(e)) from e else: # Hosted image upload url upload_url = _hosted_upload_url(api_key, project_url, image_path, split, coalesced_batch_name, tag_names) - # Get response - response = requests.post(upload_url, timeout=(300, 300)) + try: + # Get response + response = requests.post(upload_url, timeout=(300, 300)) + except RequestException as e: + raise ImageUploadError(str(e)) from e responsejson = None try: @@ -147,12 +156,15 @@ def save_annotation( api_key, project_url, annotation_name, image_id, job_name, is_prediction, overwrite ) - response = requests.post( - upload_url, - data=json.dumps({"annotationFile": annotation_string, "labelmap": annotation_labelmap}), - headers={"Content-Type": "application/json"}, - timeout=(60, 60), - ) + try: + response = requests.post( + upload_url, + data=json.dumps({"annotationFile": annotation_string, "labelmap": annotation_labelmap}), + headers={"Content-Type": "application/json"}, + timeout=(60, 60), + ) + except RequestException as e: + raise AnnotationSaveError(str(e)) from e # Handle response responsejson = None diff --git a/roboflow/core/project.py b/roboflow/core/project.py index cfad1dde..374c864f 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -11,7 +11,7 @@ import requests from roboflow.adapters import rfapi -from roboflow.adapters.rfapi import ImageUploadError +from roboflow.adapters.rfapi import AnnotationSaveError, ImageUploadError from roboflow.config import API_URL, DEMO_KEYS from roboflow.core.version import Version from roboflow.util.general import Retry @@ -515,26 +515,34 @@ def save_annotation( job_name=None, is_prediction: bool = False, annotation_overwrite=False, + num_retry_uploads=0, ): project_url = self.id.rsplit("/")[1] annotation_name, annotation_str = self._annotation_params(annotation_path) t0 = time.time() + upload_retry_attempts = 0 + retry = Retry(num_retry_uploads, AnnotationSaveError) - annotation = rfapi.save_annotation( - self.__api_key, - project_url, - annotation_name, # type: ignore[type-var] - annotation_str, # type: ignore[type-var] - image_id, - job_name=job_name, # type: ignore[type-var] - is_prediction=is_prediction, - annotation_labelmap=annotation_labelmap, - overwrite=annotation_overwrite, - ) + try: + annotation = rfapi.save_annotation( + self.__api_key, + project_url, + annotation_name, # type: ignore[type-var] + annotation_str, # type: ignore[type-var] + image_id, + job_name=job_name, # type: ignore[type-var] + is_prediction=is_prediction, + annotation_labelmap=annotation_labelmap, + overwrite=annotation_overwrite, + ) + upload_retry_attempts = retry.retries + except AnnotationSaveError as e: + e.retries = upload_retry_attempts + raise upload_time = time.time() - t0 - return annotation, upload_time + return annotation, upload_time, upload_retry_attempts def single_upload( self, @@ -563,6 +571,7 @@ def single_upload( uploaded_image, uploaded_annotation = None, None upload_time, annotation_time = None, None upload_retry_attempts = 0 + annotation_upload_retry_attempts = 0 if image_path: uploaded_image, upload_time, upload_retry_attempts = self.upload_image( @@ -579,13 +588,14 @@ def single_upload( image_id = uploaded_image["id"] # type: ignore[index] if annotation_path and image_id: - uploaded_annotation, annotation_time = self.save_annotation( + uploaded_annotation, annotation_time, annotation_upload_retry_attempts = self.save_annotation( annotation_path, annotation_labelmap, image_id, batch_name, is_prediction, annotation_overwrite, + num_retry_uploads=num_retry_uploads, ) return { @@ -594,6 +604,7 @@ def single_upload( "upload_time": upload_time, "annotation_time": annotation_time, "upload_retry_attempts": upload_retry_attempts, + "annotation_upload_retry_attempts": annotation_upload_retry_attempts, } def _annotation_params(self, annotation_path): diff --git a/roboflow/core/workspace.py b/roboflow/core/workspace.py index 815c494f..3a59b8f4 100644 --- a/roboflow/core/workspace.py +++ b/roboflow/core/workspace.py @@ -349,6 +349,7 @@ def _upload_image(imagedesc): batch_name=batch_name, sequence_number=imagedesc.get("index"), sequence_size=len(images), + num_retry_uploads=num_retries, ) return image, upload_time, upload_retry_attempts @@ -376,6 +377,7 @@ def _save_annotation(image_id, imagedesc): annotation_labelmap=labelmap, image_id=image_id, job_name=batch_name, + num_retry_uploads=num_retries, ) return annotation, upload_time diff --git a/roboflow/util/general.py b/roboflow/util/general.py index fa6a29dd..9c92e552 100644 --- a/roboflow/util/general.py +++ b/roboflow/util/general.py @@ -1,4 +1,6 @@ import sys +import time +from random import random def write_line(line): @@ -13,8 +15,16 @@ def __init__(self, max_retries, retry_on): self.retry_on = retry_on self.retries = 0 + def backoff(self): + """ + Backoff for a random time based on number of retries. + """ + base_t_ms = 100 + max_t_ms = 30000 + sleep_ms = random() * min(max_t_ms, base_t_ms * 2**self.retries) + time.sleep(int(sleep_ms) / 1000) + def __call__(self, func, *args, **kwargs): - self.retries = 0 retry_on = self.retry_on if not retry_on: retry_on = (Exception,) @@ -24,8 +34,9 @@ def __call__(self, func, *args, **kwargs): return func(*args, **kwargs) except BaseException as e: if isinstance(e, retry_on): - self.retries += 1 - if self.retries > self.max_retries: + if self.retries >= self.max_retries: raise + self.backoff() + self.retries += 1 else: raise