From 8f6daa5e2bc8f15399b654933f4ff003bf616dd7 Mon Sep 17 00:00:00 2001 From: Philipp Wiesner Date: Tue, 7 Jan 2020 15:18:16 +0100 Subject: [PATCH 1/2] Fixed bug in GCSFS.fix_storage() and prepared release of 1.1.0 --- CHANGELOG.rst | 9 +++++++++ fs_gcsfs/_gcsfs.py | 14 ++++++++------ setup.py | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fbc9955..da49b3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,12 +20,21 @@ Possible types of changes are: Unreleased ---------- + +1.1.0 - 07.01.2020 +------------------ + Added ''''' - ``GCSFS.get_mapper()`` which returns a ``GCSMap`` that wraps a ``GCSFS`` as a ``MutableMapping``. The keys of the mapping become files and the values (which must be bytes) the contents of those files. This is particularly useful to be used with libraries such as `xarray `_ or `zarr `_. +Fixed +''''' +- ``GCSFS.fix_storage()`` no longer creates a directory marker if ``root_path`` is the actual root of the bucket. + Apart from not having any advantage, this caused subsequent ``GCSFS.fix_storage()`` calls as well as ``GCSFS.walk()`` to be stuck in endless loops. + 1.0.0 - 27.08.2019 ------------------ diff --git a/fs_gcsfs/_gcsfs.py b/fs_gcsfs/_gcsfs.py index 738008e..41f2217 100644 --- a/fs_gcsfs/_gcsfs.py +++ b/fs_gcsfs/_gcsfs.py @@ -6,7 +6,7 @@ import os import tempfile import mimetypes -from typing import Optional, List, Union, Tuple, Iterator, MutableMapping +from typing import Optional, List, Union, Tuple, Iterator, MutableMapping, Any import google from fs import ResourceType, errors, tools @@ -477,7 +477,9 @@ def fix_storage(self) -> None: # TODO test while name != self.root_path: all_dirs.add(name) name = dirname(name) - all_dirs.add(self.root_path) + + if forcedir(self.root_path) != "/": + all_dirs.add(self.root_path) unmarked_dirs = all_dirs.difference(marked_dirs) logger.info("{} directories in total".format(len(all_dirs))) @@ -632,17 +634,17 @@ class GCSMap(MutableMapping): def __init__(self, gcsfs: GCSFS): self.gcsfs = gcsfs - def __getitem__(self, key: str) -> bytes: + def __getitem__(self, key: Any) -> bytes: try: return self.gcsfs.getbytes(str(key)) except errors.ResourceNotFound: raise KeyError(key) - def __setitem__(self, key: str, value: bytes): + def __setitem__(self, key: Any, value: Any) -> None: self.gcsfs.makedirs(dirname(str(key)), recreate=True) self.gcsfs.setbytes(str(key), bytes(value)) - def __delitem__(self, key): + def __delitem__(self, key) -> None: self.gcsfs.remove(str(key)) def __iter__(self) -> Iterator[str]: @@ -651,7 +653,7 @@ def __iter__(self) -> Iterator[str]: def __len__(self) -> int: return sum(1 for _ in self.keys()) - def __contains__(self, key: str) -> bool: + def __contains__(self, key: Any) -> bool: return self.gcsfs.exists(str(key)) def keys(self) -> Iterator[str]: diff --git a/setup.py b/setup.py index 5781871..0cdefc7 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation", "Topic :: System :: Filesystems", ], From 072c548f47290441d9fe8c2c36596984b1984686 Mon Sep 17 00:00:00 2001 From: Philipp Wiesner Date: Tue, 7 Jan 2020 16:52:27 +0100 Subject: [PATCH 2/2] Added resolved issues to Changelog --- CHANGELOG.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da49b3a..657cf00 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,12 +28,12 @@ Added ''''' - ``GCSFS.get_mapper()`` which returns a ``GCSMap`` that wraps a ``GCSFS`` as a ``MutableMapping``. The keys of the mapping become files and the values (which must be bytes) the contents of those files. - This is particularly useful to be used with libraries such as `xarray `_ or `zarr `_. + This is particularly useful to be used with libraries such as `xarray `_ or `zarr `_. (#21) Fixed ''''' - ``GCSFS.fix_storage()`` no longer creates a directory marker if ``root_path`` is the actual root of the bucket. - Apart from not having any advantage, this caused subsequent ``GCSFS.fix_storage()`` calls as well as ``GCSFS.walk()`` to be stuck in endless loops. + Apart from not having any advantage, this caused subsequent ``GCSFS.fix_storage()`` calls as well as ``GCSFS.walk()`` to be stuck in endless loops. (#19) 1.0.0 - 27.08.2019 @@ -42,7 +42,7 @@ Fixed Changed ''''''' - Instead of uploading all blobs as *application/octet-stream*, the MIME type is now guessed via ``mimetypes.guess_type()``. - This enables e.g. hotlinking images directly from GCS. + This enables e.g. hotlinking images directly from GCS. (#15) 0.4.2 - 30.07.2019 @@ -50,7 +50,7 @@ Changed Fixed ''''' -- Fixed a bug where the url parameter ``strict`` was not considered by GCSFS, e.g. in ``open_fs("gs://bucket_name?strict=False")`` +- Fixed a bug where the url parameter ``strict`` was not considered by GCSFS, e.g. in ``open_fs("gs://bucket_name?strict=False")`` (#11) 0.4.1 - 18.12.2018 @@ -58,7 +58,7 @@ Fixed Fixed ''''' -- Fixed a bug where ``create=True`` in combination with an "empty-ish ``root_path`` like ``""``, ``"."`` or ``"/"`` would create a directory marker. +- Fixed a bug where ``create=True`` in combination with an empty-ish ``root_path`` like ``""``, ``"."`` or ``"/"`` would create a directory marker. 0.4.0 - 11.12.2018 @@ -67,7 +67,7 @@ Fixed Added ''''' - Implemented the ``create`` property on ``GCSFS`` and the corresponding opener. By default all new GCSFS instances have ``create=False`` (PyFilesystem default) - which means they will raise a ``CreateFailed`` exception if ``root_path`` does not exist + which means they will raise a ``CreateFailed`` exception if ``root_path`` does not exist (#8) 0.3.0 - 20.11.2018