From 7eb4354186a546a793d60956a70c17d4fc59b175 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 13 May 2024 10:58:37 -0400 Subject: [PATCH 1/8] Updating to implement new storage manager interface --- guillotina_gcloudstorage/storage.py | 73 ++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index ae800dd..bccba0e 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import traceback from async_lru import alru_cache from datetime import datetime from datetime import timedelta @@ -24,11 +25,12 @@ from guillotina.utils import get_authenticated_user_id from guillotina.utils import get_current_request from guillotina.utils import to_str +from guillotina.interfaces.files import ICloudBlob from guillotina_gcloudstorage.interfaces import IGCloudBlobStore from guillotina_gcloudstorage.interfaces import IGCloudFile from guillotina_gcloudstorage.interfaces import IGCloudFileField from oauth2client.service_account import ServiceAccountCredentials -from typing import AsyncIterator +from typing import AsyncIterator, List, Optional, Tuple from urllib.parse import quote_plus from zope.interface import implementer @@ -102,10 +104,16 @@ def dictfile_converter(value, field): return GCloudFile(**value) -@implementer(IGCloudFile) +@implementer(IGCloudFile, ICloudBlob) class GCloudFile(BaseCloudFile): """File stored in a GCloud, with a filename.""" + def __init__(self, key: str, bucket: str, size: int, createdTime: Optional[datetime]): + self.key = key + self.bucket = bucket + self.size = size + self.createdTime = createdTime + def _is_uploaded_file(file): return file is not None and isinstance(file, GCloudFile) and file.uri is not None @@ -610,3 +618,64 @@ async def generate_download_signed_url( if credentials: request_args["credentials"] = credentials return blob.generate_signed_url(**request_args) + + + async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_keys=1000) -> Tuple[List[ICloudBlob], str]: + """ + Get a page of items from the bucket + """ + page = await self.iterate_bucket_page(page_token, prefix) + blobs = [ + GCloudFile( + key = item.get("name"), + bucket = item.get("bucket"), + createdTime = item.get("timeCreated"), + size = item.get("size") + ) + for item + in page.get("items", []) + ] + next_page_token = page.get("nextPageToken") + + return blobs, next_page_token + + + async def delete_blobs(self, keys: List[str], bucket_name: Optional[str] = None) -> Tuple[List[str], List[str]]: + """ + Deletes a batch of files. Returns successful and failed keys. + """ + client = self.get_client() + + if not bucket_name: + bucket_name = await self.get_bucket_name() + + bucket = client.bucket(bucket_name) + + with client.batch(raise_exception=False) as batch: + for key in keys: + bucket.delete_blob(key) + + success_keys = [] + failed_keys = [] + # for response in batch._responses: + # # key = response + # key=1 + # if 200 <= response.status_code <= 300: + # success_keys.append(key) + # else: + # failed_keys.append(key) + + return success_keys, failed_keys + + + async def delete_bucket(self, bucket_name: Optional[str] = None): + """ + Delete the given bucket + """ + client = self.get_client() + + if not bucket_name: + bucket_name = await self.get_bucket_name() + + bucket = client.bucket(bucket_name) + bucket.delete(force=True) From c2ad0fffa06e6fcad120cbd474de22d35c7a381d Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 14 May 2024 13:06:56 -0400 Subject: [PATCH 2/8] Updating to raise exception for deletion failure --- guillotina_gcloudstorage/storage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index bccba0e..75a2817 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -6,6 +6,7 @@ from functools import lru_cache from guillotina import configure from guillotina import task_vars +from guillotina.db.exceptions import DeleteStorageException from guillotina.component import get_multi_adapter from guillotina.component import get_utility from guillotina.exceptions import FileNotFoundException @@ -678,4 +679,8 @@ async def delete_bucket(self, bucket_name: Optional[str] = None): bucket_name = await self.get_bucket_name() bucket = client.bucket(bucket_name) - bucket.delete(force=True) + + try: + bucket.delete(force=True) + except ValueError: + raise DeleteStorageException() From a29362daf20b044aab6e8c4a098c2803fd944c6a Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 14 May 2024 14:29:20 -0400 Subject: [PATCH 3/8] Standardizing name property on blobs --- guillotina_gcloudstorage/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index 75a2817..158a84e 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -628,10 +628,10 @@ async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_key page = await self.iterate_bucket_page(page_token, prefix) blobs = [ GCloudFile( - key = item.get("name"), + name = item.get("name"), bucket = item.get("bucket"), createdTime = item.get("timeCreated"), - size = item.get("size") + size = int(item.get("size")) ) for item in page.get("items", []) From fd53b5bd64a63d4f25930d550e2cb5055260c6de Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 14 May 2024 19:01:40 -0400 Subject: [PATCH 4/8] Splitting out blob metdata from file --- guillotina_gcloudstorage/storage.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index 158a84e..39c227c 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -26,7 +26,7 @@ from guillotina.utils import get_authenticated_user_id from guillotina.utils import get_current_request from guillotina.utils import to_str -from guillotina.interfaces.files import ICloudBlob +from guillotina.files.field import BlobMetadata from guillotina_gcloudstorage.interfaces import IGCloudBlobStore from guillotina_gcloudstorage.interfaces import IGCloudFile from guillotina_gcloudstorage.interfaces import IGCloudFileField @@ -105,16 +105,10 @@ def dictfile_converter(value, field): return GCloudFile(**value) -@implementer(IGCloudFile, ICloudBlob) +@implementer(IGCloudFile) class GCloudFile(BaseCloudFile): """File stored in a GCloud, with a filename.""" - def __init__(self, key: str, bucket: str, size: int, createdTime: Optional[datetime]): - self.key = key - self.bucket = bucket - self.size = size - self.createdTime = createdTime - def _is_uploaded_file(file): return file is not None and isinstance(file, GCloudFile) and file.uri is not None @@ -621,13 +615,13 @@ async def generate_download_signed_url( return blob.generate_signed_url(**request_args) - async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_keys=1000) -> Tuple[List[ICloudBlob], str]: + async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_keys=1000) -> Tuple[List[BlobMetadata], str]: """ Get a page of items from the bucket """ page = await self.iterate_bucket_page(page_token, prefix) blobs = [ - GCloudFile( + BlobMetadata( name = item.get("name"), bucket = item.get("bucket"), createdTime = item.get("timeCreated"), @@ -636,7 +630,7 @@ async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_key for item in page.get("items", []) ] - next_page_token = page.get("nextPageToken") + next_page_token = page.get("nextPageToken", None) return blobs, next_page_token From 61e16fcf466a2ddaceb3e6321d1b342ce2c202cd Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 15 May 2024 10:48:36 -0400 Subject: [PATCH 5/8] Updating to implemet vacuum interface --- guillotina_gcloudstorage/storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index 39c227c..c5cfc92 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -27,6 +27,7 @@ from guillotina.utils import get_current_request from guillotina.utils import to_str from guillotina.files.field import BlobMetadata +from guillotina.interfaces.files import IBlobVacuum from guillotina_gcloudstorage.interfaces import IGCloudBlobStore from guillotina_gcloudstorage.interfaces import IGCloudFile from guillotina_gcloudstorage.interfaces import IGCloudFileField @@ -433,6 +434,7 @@ async def get_access_token(): return await _get_access_token(round(time.time() / 300)) +@implementer(IBlobVacuum) class GCloudBlobStore(object): def __init__(self, settings, loop=None): self._loop = loop From 1e1def8c85ed7aab02f65b7a4d9fe2d776e61259 Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 15 May 2024 14:30:38 -0400 Subject: [PATCH 6/8] Adding datetime parsing --- guillotina_gcloudstorage/storage.py | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index c5cfc92..ea0c849 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -3,6 +3,7 @@ from async_lru import alru_cache from datetime import datetime from datetime import timedelta +from dateutil.parser import parse from functools import lru_cache from guillotina import configure from guillotina import task_vars @@ -626,7 +627,7 @@ async def get_blobs(self, page_token: Optional[str] = None, prefix=None, max_key BlobMetadata( name = item.get("name"), bucket = item.get("bucket"), - createdTime = item.get("timeCreated"), + createdTime = parse(item.get("timeCreated")), size = int(item.get("size")) ) for item diff --git a/setup.py b/setup.py index 6178da4..12a0dbb 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ "ujson", "backoff", "async-lru", + "zope-interface<6,>=5.0.0" ], extras_require={"test": test_reqs}, tests_require=test_reqs, From 58e75332b9f4bdc7e09f3e715e01ad30ad3f5e76 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 16 May 2024 09:00:45 -0400 Subject: [PATCH 7/8] Updating to append keys by status --- guillotina_gcloudstorage/storage.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/guillotina_gcloudstorage/storage.py b/guillotina_gcloudstorage/storage.py index ea0c849..ee1ebfd 100644 --- a/guillotina_gcloudstorage/storage.py +++ b/guillotina_gcloudstorage/storage.py @@ -655,13 +655,12 @@ async def delete_blobs(self, keys: List[str], bucket_name: Optional[str] = None) success_keys = [] failed_keys = [] - # for response in batch._responses: - # # key = response - # key=1 - # if 200 <= response.status_code <= 300: - # success_keys.append(key) - # else: - # failed_keys.append(key) + for idx, response in enumerate(batch._responses): + key=keys[idx] + if 200 <= response.status_code <= 300: + success_keys.append(key) + else: + failed_keys.append(key) return success_keys, failed_keys From ec737704c546937dcb2a371dc5704a148d65cd0d Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 17 May 2024 10:01:35 -0400 Subject: [PATCH 8/8] Updating version and changelog --- CHANGELOG.rst | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9b1abab..b1fe4e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,7 @@ +6.1.0 (unreleased) +------------------- +- Updating cloud vacuum support to standardize implementation + 6.0.12 (unreleased) ------------------- - More auth retries diff --git a/VERSION b/VERSION index 9eaadd7..dfda3e0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.12 +6.1.0