Beta version of retry mechanism

CHANGELOG.rst

@@ -20,6 +20,15 @@ Possible types of changes are:
 Unreleased
 ----------
 
+Added
+'''''
+- Experimental: Clients now automatically retry GCS requests that return with "503 Service Unavailable" or "504 Gateway Timeout.
+
+Fixed
+'''''
+- Some tests were still calling ``get_bucket()`` from the constructor of ``GCSFS``.
+
+
 1.3.0 - 20.05.2020
 ------------------
 

fs_gcsfs/_gcsfs.py

@@ -19,6 +19,9 @@
 from fs.time import datetime_to_epoch
 from google.cloud.storage import Client
 from google.cloud.storage.blob import Blob
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
 
 __all__ = ["GCSFS"]
 
@@ -57,6 +60,7 @@ def __init__(self,
                  root_path: str = None,
                  create: bool = False,
                  client: Client = None,
+                 retry: int = 5,
                  strict: bool = True):
         super().__init__()
         self._bucket_name = bucket_name
@@ -70,6 +74,9 @@ def __init__(self,
         self.client = client
         if self.client is None:
             self.client = Client()
+            if retry:
+                adapter = HTTPAdapter(max_retries=Retry(total=retry, status_forcelist=[429, 503, 504], method_whitelist=False))
+                self.client._http.mount("https://", adapter)
 
         self.bucket = self.client.bucket(self._bucket_name)
 

fs_gcsfs/tests/conftest.py

@@ -14,7 +14,7 @@ def client():
 
 @pytest.fixture(scope="module")
 def bucket(client):
-    return client.get_bucket(os.environ['TEST_BUCKET'])
+    return client.bucket(os.environ['TEST_BUCKET'])
 
 
 @pytest.fixture(scope="function")

fs_gcsfs/tests/test_gcsfs.py

@@ -18,7 +18,7 @@ class TestGCSFS(FSTestCases, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.client = Client()
-        cls.bucket = cls.client.get_bucket(TEST_BUCKET)
+        cls.bucket = cls.client.bucket(TEST_BUCKET)
         super().setUpClass()
 
     def setUp(self):