diff --git a/keg_elements/sentry.py b/keg_elements/sentry.py index d2e87f0..6c965c7 100644 --- a/keg_elements/sentry.py +++ b/keg_elements/sentry.py @@ -13,6 +13,11 @@ except ImportError: requests = None +try: + import urllib3 +except ImportError: + urllib3 = None + try: import yaml except ImportError: @@ -249,6 +254,16 @@ def install_sentry(app, integrations, release=None, event_filter=None, **kwargs) ) +def retry_http_adapter(): + retry_strategy = urllib3.util.retry.Retry( + total=3, + backoff_factor=1, + allowed_methods=['POST', 'PUT'], + status_forcelist=[429, 500, 502, 503, 504], + ) + return requests.sessions.HTTPAdapter(max_retries=retry_strategy) + + class SentryMonitorError(Exception): pass @@ -344,18 +359,23 @@ def url(self): return base def make_request(self, payload): - method = requests.put if self.checkin_id else requests.post - resp = method( - self.url, - json=payload, - headers={ - 'Authorization': f'DSN {self.dsn}', - }, - timeout=10, - ) - if resp.status_code >= 400: - raise SentryMonitorError(f'{resp.status_code}: {resp.content}') - return resp.json() + with requests.Session() as session: + session.mount('https://', retry_http_adapter()) + method = session.put if self.checkin_id else session.post + try: + resp = method( + self.url, + json=payload, + headers={ + 'Authorization': f'DSN {self.dsn}', + }, + timeout=10, + ) + if resp.status_code >= 400: + raise SentryMonitorError(f'{resp.status_code}: {resp.content}') + return resp.json() + except requests.exceptions.RetryError as e: + raise SentryMonitorError(str(e)) def ping_status(self, status): self.status = status diff --git a/keg_elements/tests/test_sentry.py b/keg_elements/tests/test_sentry.py index a1fba81..9066919 100644 --- a/keg_elements/tests/test_sentry.py +++ b/keg_elements/tests/test_sentry.py @@ -425,11 +425,11 @@ def test_job_network_error(self): matchers.json_params_matcher({'status': 'in_progress', 'environment': 'testenv'}), ], ) - with pytest.raises(sentry.SentryMonitorError, match='something wrong'): + with pytest.raises(sentry.SentryMonitorError, match='Max retries exceeded'): with sentry.sentry_monitor_job('myorg', 'monitor-key', do_ping=True): pass - assert resp_in_progress.call_count == 1 + assert resp_in_progress.call_count == 4 @responses.activate @mock.patch.dict('flask.current_app.config', {