Add retries for Sentry monitor API calls

Refs GH-192
level12 · Oct 19, 2023 · afd9ca9 · afd9ca9
1 parent aa60cd6
commit afd9ca9
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 14 deletions.
diff --git a/keg_elements/sentry.py b/keg_elements/sentry.py
@@ -13,6 +13,11 @@
 except ImportError:
     requests = None
 
+try:
+    import urllib3
+except ImportError:
+    urllib3 = None
+
 try:
     import yaml
 except ImportError:
@@ -249,6 +254,16 @@ def install_sentry(app, integrations, release=None, event_filter=None, **kwargs)
     )
 
 
+def retry_http_adapter():
+    retry_strategy = urllib3.util.retry.Retry(
+        total=3,
+        backoff_factor=1,
+        allowed_methods=['POST', 'PUT'],
+        status_forcelist=[429, 500, 502, 503, 504],
+    )
+    return requests.sessions.HTTPAdapter(max_retries=retry_strategy)
+
+
 class SentryMonitorError(Exception):
     pass
 
@@ -344,18 +359,23 @@ def url(self):
         return base
 
     def make_request(self, payload):
-        method = requests.put if self.checkin_id else requests.post
-        resp = method(
-            self.url,
-            json=payload,
-            headers={
-                'Authorization': f'DSN {self.dsn}',
-            },
-            timeout=10,
-        )
-        if resp.status_code >= 400:
-            raise SentryMonitorError(f'{resp.status_code}: {resp.content}')
-        return resp.json()
+        with requests.Session() as session:
+            session.mount('https://', retry_http_adapter())
+            method = session.put if self.checkin_id else session.post
+            try:
+                resp = method(
+                    self.url,
+                    json=payload,
+                    headers={
+                        'Authorization': f'DSN {self.dsn}',
+                    },
+                    timeout=10,
+                )
+                if resp.status_code >= 400:
+                    raise SentryMonitorError(f'{resp.status_code}: {resp.content}')
+                return resp.json()
+            except requests.exceptions.RetryError as e:
+                raise SentryMonitorError(str(e))
 
     def ping_status(self, status):
         self.status = status

diff --git a/keg_elements/tests/test_sentry.py b/keg_elements/tests/test_sentry.py
@@ -425,11 +425,11 @@ def test_job_network_error(self):
                 matchers.json_params_matcher({'status': 'in_progress', 'environment': 'testenv'}),
             ],
         )
-        with pytest.raises(sentry.SentryMonitorError, match='something wrong'):
+        with pytest.raises(sentry.SentryMonitorError, match='Max retries exceeded'):
             with sentry.sentry_monitor_job('myorg', 'monitor-key', do_ping=True):
                 pass
 
-        assert resp_in_progress.call_count == 1
+        assert resp_in_progress.call_count == 4
 
     @responses.activate
     @mock.patch.dict('flask.current_app.config', {