diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e5fbc67..db0f9c3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,15 @@ Possible types of changes are: Unreleased ---------- +Added +''''' +- Experimental: Clients now automatically retry GCS requests that return with "503 Service Unavailable" or "504 Gateway Timeout. + +Fixed +''''' +- Some tests were still calling ``get_bucket()`` from the constructor of ``GCSFS``. + + 1.3.0 - 20.05.2020 ------------------ diff --git a/fs_gcsfs/_gcsfs.py b/fs_gcsfs/_gcsfs.py index c8d955d..d611fa1 100644 --- a/fs_gcsfs/_gcsfs.py +++ b/fs_gcsfs/_gcsfs.py @@ -19,6 +19,9 @@ from fs.time import datetime_to_epoch from google.cloud.storage import Client from google.cloud.storage.blob import Blob +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + __all__ = ["GCSFS"] @@ -57,6 +60,7 @@ def __init__(self, root_path: str = None, create: bool = False, client: Client = None, + retry: int = 5, strict: bool = True): super().__init__() self._bucket_name = bucket_name @@ -70,6 +74,9 @@ def __init__(self, self.client = client if self.client is None: self.client = Client() + if retry: + adapter = HTTPAdapter(max_retries=Retry(total=retry, status_forcelist=[429, 503, 504], method_whitelist=False)) + self.client._http.mount("https://", adapter) self.bucket = self.client.bucket(self._bucket_name) diff --git a/fs_gcsfs/tests/conftest.py b/fs_gcsfs/tests/conftest.py index d7872ff..2760730 100644 --- a/fs_gcsfs/tests/conftest.py +++ b/fs_gcsfs/tests/conftest.py @@ -14,7 +14,7 @@ def client(): @pytest.fixture(scope="module") def bucket(client): - return client.get_bucket(os.environ['TEST_BUCKET']) + return client.bucket(os.environ['TEST_BUCKET']) @pytest.fixture(scope="function") diff --git a/fs_gcsfs/tests/test_gcsfs.py b/fs_gcsfs/tests/test_gcsfs.py index 9e41e2c..9239735 100644 --- a/fs_gcsfs/tests/test_gcsfs.py +++ b/fs_gcsfs/tests/test_gcsfs.py @@ -18,7 +18,7 @@ class TestGCSFS(FSTestCases, unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Client() - cls.bucket = cls.client.get_bucket(TEST_BUCKET) + cls.bucket = cls.client.bucket(TEST_BUCKET) super().setUpClass() def setUp(self):