implemented "create" property on GCSFS and opener (#9)

* implemented "create" property on GCSFS and opener

* flake8

* minor bugfix

* PR comments

CHANGELOG.rst

@@ -21,6 +21,16 @@ Unreleased
 ----------
 
 
+0.4.0 - 11.12.2018
+------------------
+
+Added
+'''''
+- Implemented the ``create`` property on ``GCSFS`` and the corresponding opener. By default all new GCSFS instances have ``create=False`` (PyFilesystem default)
+    which means they will raise a ``CreateFailed`` exception if ``root_path`` does not exist
+
+
+
 0.3.0 - 20.11.2018
 ------------------
 

fs_gcsfs/_gcsfs.py

@@ -10,6 +10,7 @@
 import google
 from fs import ResourceType, errors, tools
 from fs.base import FS
+from fs.errors import CreateFailed
 from fs.info import Info
 from fs.mode import Mode
 from fs.path import basename, dirname, forcedir, normpath, relpath, join
@@ -32,9 +33,11 @@ class GCSFS(FS):
     Args:
         bucket_name: The GCS bucket name.
         root_path: The root directory within the GCS Bucket
+        create: Whether to create ``root_path`` on initialization or not. If ``root_path`` does not yet exist and ``create=False`` a ``CreateFailed``
+            exception will be raised. To disable ``root_path`` validation entirely set ``strict=False``.
         client: A :class:`google.storage.Client` exposing the google storage API.
         strict: When ``True`` (default) GCSFS will follow the PyFilesystem specification exactly. Set to ``False`` to disable validation of destination paths
-                which may speed up uploads / downloads.
+            which may speed up some operations.
     """
 
     _meta = {
@@ -52,8 +55,10 @@ class GCSFS(FS):
     def __init__(self,
                  bucket_name: str,
                  root_path: str = None,
+                 create: bool = False,
                  client: Client = None,
                  strict: bool = True):
+        super().__init__()
         self._bucket_name = bucket_name
         if not root_path:
             root_path = ""
@@ -66,8 +71,20 @@ def __init__(self,
         if self.client is None:
             self.client = Client()
 
-        self.bucket = self.client.get_bucket(self._bucket_name)
-        super(GCSFS, self).__init__()
+        try:
+            self.bucket = self.client.get_bucket(self._bucket_name)
+        except google.api_core.exceptions.NotFound as err:
+            raise CreateFailed("The bucket \"{}\" does not seem to exist".format(self._bucket_name)) from err
+        except google.api_core.exceptions.Forbidden as err:
+            raise CreateFailed("You don't have access to the bucket \"{}\"".format(self._bucket_name)) from err
+
+        if create:
+            root_marker = self._get_blob(forcedir(root_path))
+            if root_marker is None:
+                blob = self.bucket.blob(forcedir(root_path))
+                blob.upload_from_string(b"")
+        elif strict and self._get_blob(forcedir(root_path)) is None:
+            raise errors.CreateFailed("Root path \"{}\" does not exist".format(root_path))
 
     def __repr__(self) -> str:
         return _make_repr(

fs_gcsfs/_version.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"

fs_gcsfs/opener.py

@@ -17,9 +17,9 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd):  # pylint: disa
         path_parts = iteratepath(parse_result.resource)
 
         bucket_name = path_parts[0]
-        dir_path = join(*path_parts[1:])
+        root_path = join(*path_parts[1:])
 
         if not bucket_name:
             raise OpenerError("invalid bucket name in '{}'".format(fs_url))
 
-        return GCSFS(bucket_name, dir_path)
+        return GCSFS(bucket_name, root_path=root_path, create=create)

fs_gcsfs/tests/test_gcsfs.py

@@ -1,9 +1,9 @@
+import os
 import unittest
 import uuid
-import os
 
 import pytest
-from fs.errors import IllegalBackReference
+from fs.errors import IllegalBackReference, CreateFailed
 from fs.test import FSTestCases
 from google.cloud.storage import Client
 
@@ -30,7 +30,7 @@ def tearDown(self):
             blob.delete()
 
     def make_fs(self):
-        return GCSFS(bucket_name=TEST_BUCKET, root_path=self.root_path, client=self.client)
+        return GCSFS(bucket_name=TEST_BUCKET, root_path=self.root_path, client=self.client, create=True)
 
 
 @pytest.fixture(scope="module")
@@ -56,7 +56,7 @@ def bucket(client):
 def tmp_gcsfs(bucket, client):
     """Yield a temporary `GCSFS` at a unique 'root-blob' within the test bucket."""
     path = "gcsfs/" + str(uuid.uuid4())
-    yield GCSFS(bucket_name=bucket.name, root_path=path, client=client)
+    yield GCSFS(bucket_name=bucket.name, root_path=path, client=client, create=True)
     for blob in bucket.list_blobs(prefix=path):
         blob.delete()
 
@@ -82,24 +82,24 @@ def tmp_gcsfs(bucket, client):
     ("foo/../bar", "root_path", "root_path/bar"),
 ])
 def test_path_to_key(path, root_path, expected, client_mock):
-    gcs_fs = GCSFS(bucket_name="bucket", root_path=root_path, client=client_mock)
+    gcs_fs = GCSFS(bucket_name=TEST_BUCKET, root_path=root_path, client=client_mock, strict=False)
     assert gcs_fs._path_to_key(path) == expected
     assert gcs_fs._path_to_dir_key(path) == expected + GCSFS.DELIMITER
 
 
 def test_path_to_key_fails_if_path_is_parent_of_root_path(client_mock):
-    gcs_fs = GCSFS(bucket_name="bucket", client=client_mock)
+    gcs_fs = GCSFS(bucket_name=TEST_BUCKET, client=client_mock, strict=False)
     with pytest.raises(IllegalBackReference):
         gcs_fs._path_to_key("..")
 
-    gcs_fs_with_root_path = GCSFS(bucket_name="bucket", root_path="root_path", client=client_mock)
+    gcs_fs_with_root_path = GCSFS(bucket_name="bucket", root_path="root_path", client=client_mock, strict=False)
     with pytest.raises(IllegalBackReference):
         gcs_fs_with_root_path._path_to_key("..")
 
 
 def test_listdir_works_on_bucket_as_root_directory(client):
     """Regression test for a bug fixed in 0.2.1"""
-    gcs_fs = GCSFS(bucket_name=TEST_BUCKET, client=client)
+    gcs_fs = GCSFS(bucket_name=TEST_BUCKET, client=client, create=True)
 
     blob = str(uuid.uuid4())
     directory = str(uuid.uuid4())
@@ -144,3 +144,13 @@ def test_fix_storage_does_not_overwrite_existing_directory_markers_with_custom_c
     tmp_gcsfs.fix_storage()
 
     assert blob.download_as_string() == content
+
+
+def test_instantiation_fails_if_no_access_to_bucket():
+    with pytest.raises(CreateFailed):
+        GCSFS(bucket_name=str(uuid.uuid4()))
+
+
+def test_instantiation_with_create_false_fails_for_non_existing_root_path():
+    with pytest.raises(CreateFailed):
+        GCSFS(bucket_name=TEST_BUCKET, root_path=str(uuid.uuid4()), create=False)