diff --git a/indra/literature/pubmed_client.py b/indra/literature/pubmed_client.py index b52eaf3e14..7971c766b6 100644 --- a/indra/literature/pubmed_client.py +++ b/indra/literature/pubmed_client.py @@ -2,6 +2,8 @@ Search and get metadata for articles in Pubmed. """ import logging +import random + import requests from time import sleep from typing import List @@ -19,7 +21,7 @@ # Send request can't be cached by lru_cache because it takes a dict # (a mutable/unhashable type) as an argument. We cache the callers instead. -def send_request(url, data): +def send_request(url, data, retry_pause=0.5, max_tries=3): try: res = requests.get(url, params=data) except requests.exceptions.Timeout as e: @@ -32,9 +34,12 @@ def send_request(url, data): logger.error('url: %s, data: %s' % (url, data)) logger.error(e) return None - if res.status_code == 429: - sleep(0.5) - res = requests.get(url, params=data) + if res.status_code in {429, 502, 503} and max_tries > 0: + sleep(retry_pause) + # Increase the sleep time at random to avoid multiple clients + # retrying at the same time for e.g. tests + retry_pause += 1 + random.random() + return send_request(url, data, retry_pause, max_tries - 1) if not res.status_code == 200: logger.error('Got return code %d from pubmed client.' % res.status_code) diff --git a/indra/tests/test_db_rest.py b/indra/tests/test_db_rest.py index d06690b7aa..62e8a48326 100644 --- a/indra/tests/test_db_rest.py +++ b/indra/tests/test_db_rest.py @@ -1,4 +1,3 @@ -import random import unittest from datetime import datetime from time import sleep @@ -58,11 +57,8 @@ def test_bigger_request(): @pytest.mark.nonpublic -def test_timeout_no_persist_agent(): - candidates = ['TP53', 'NFkappaB@FPLX', 'AKT@FPLX'] - agent = random.choice(candidates) - print(agent) - resp = dbr.get_statements(agents=[agent], persist=False, timeout=0) +def test_timeout_no_persist_nfkb(): + resp = dbr.get_statements(agents=["NFkappaB@FPLX"], persist=False, timeout=0) assert resp.is_working(), "Lookup resolved too fast." resp.wait_until_done(70) assert len(resp.statements) > 0.9*EXPECTED_BATCH_SIZE, len(resp.statements) @@ -70,11 +66,10 @@ def test_timeout_no_persist_agent(): @pytest.mark.nonpublic def test_timeout_no_persist_type_object(): - candidates = ['TP53', 'NFkappaB@FPLX', 'AKT@FPLX'] - agent = random.choice(candidates) - print(agent) - resp = dbr.get_statements(stmt_type='phosphorylation', object=agent, - persist=False, timeout=0) + resp = dbr.get_statements(stmt_type='phosphorylation', + object="NFkappaB@FPLX", + persist=False, + timeout=0) assert resp.is_working(), "Lookup resolved too fast." resp.wait_until_done(70) assert len(resp.statements) > 0.9*EXPECTED_BATCH_SIZE, len(resp.statements)