From 56271e16c02d7c8ad73d9873f4d7c8d42f816c75 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Fri, 6 Oct 2023 22:20:02 +0530 Subject: [PATCH] [gs] Add Google Storage filebrowser support in Hue (#3455) What changes were proposed in this pull request? - Adding Google Storage Filebrowser in Hue as a separate Filebrowser piggybacking on S3 Filesystem implementation. - Some changes in boto package were also required to make the implementation work. How was this patch tested? - Manually. - Added new unit tests. --- apps/filebrowser/src/filebrowser/api.py | 3 + apps/filebrowser/src/filebrowser/forms.py | 3 + .../filebrowser/templates/fb_components.mako | 6 + .../src/filebrowser/templates/listdir.mako | 17 +- .../templates/listdir_components.mako | 56 ++- apps/filebrowser/src/filebrowser/views.py | 14 +- desktop/conf.dist/hue.ini | 5 +- desktop/conf/pseudo-distributed.ini.tmpl | 15 +- desktop/core/src/desktop/conf.py | 7 + desktop/core/src/desktop/js/api/apiHelper.js | 73 ++- desktop/core/src/desktop/js/api/urls.js | 1 + .../autocomplete/AutocompleteResults.ts | 3 + .../js/components/sidebar/HueSidebar.vue | 2 + .../js/jquery/plugins/jquery.filechooser.js | 17 + .../js/ko/bindings/ace/ko.aceEditor.js | 10 + .../components/assist/assistStorageEntry.js | 4 + .../js/ko/components/assist/ko.assistPanel.js | 1 + .../assist/ko.assistStoragePanel.js | 1 + .../ko/components/ko.editorDroppableMenu.js | 1 + .../src/desktop/js/sql/autocompleteResults.js | 3 + .../core/src/desktop/lib/fs/gc/__init__.py | 99 ++++ desktop/core/src/desktop/lib/fs/gc/client.py | 13 +- desktop/core/src/desktop/lib/fs/gc/gs.py | 471 ++++++++++++++++++ desktop/core/src/desktop/lib/fs/gc/gsfile.py | 50 ++ .../core/src/desktop/lib/fs/gc/gsfile_test.py | 40 ++ desktop/core/src/desktop/lib/fs/gc/gsstat.py | 100 ++++ .../core/src/desktop/lib/fs/gc/gsstat_test.py | 80 +++ desktop/core/src/desktop/lib/fs/gc/upload.py | 175 +++++++ .../core/src/desktop/lib/fs/gc/upload_test.py | 65 +++ .../src/desktop/lib/fs/ozone/upload_test.py | 6 + desktop/core/src/desktop/models.py | 10 + desktop/core/src/desktop/settings.py | 5 +- desktop/libs/aws/src/aws/s3/s3fs.py | 5 +- 33 files changed, 1316 insertions(+), 45 deletions(-) create mode 100644 desktop/core/src/desktop/lib/fs/gc/gs.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/gsfile.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/gsfile_test.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/gsstat.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/gsstat_test.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/upload.py create mode 100644 desktop/core/src/desktop/lib/fs/gc/upload_test.py diff --git a/apps/filebrowser/src/filebrowser/api.py b/apps/filebrowser/src/filebrowser/api.py index f4faddfd041..e8b0c80b3d3 100644 --- a/apps/filebrowser/src/filebrowser/api.py +++ b/apps/filebrowser/src/filebrowser/api.py @@ -21,6 +21,7 @@ from desktop.lib import fsmanager from desktop.lib.i18n import smart_unicode from desktop.lib.fs.ozone.ofs import get_ofs_home_directory +from desktop.lib.fs.gc.gs import get_gs_home_directory from azure.abfs.__init__ import get_home_dir_for_abfs from aws.s3.s3fs import get_s3_home_directory @@ -66,6 +67,8 @@ def get_filesystems_with_home_dirs(request): # Using as a public API only for no user_home_dir = request.user.get_home_directory() elif fs == 's3a': user_home_dir = get_s3_home_directory(request.user) + elif fs == 'gs': + user_home_dir = get_gs_home_directory(request.user) elif fs == 'abfs': user_home_dir = get_home_dir_for_abfs(request.user) elif fs == 'ofs': diff --git a/apps/filebrowser/src/filebrowser/forms.py b/apps/filebrowser/src/filebrowser/forms.py index d024dd47443..7d75b975319 100644 --- a/apps/filebrowser/src/filebrowser/forms.py +++ b/apps/filebrowser/src/filebrowser/forms.py @@ -30,6 +30,7 @@ from aws.s3 import S3A_ROOT, normpath as s3_normpath from azure.abfs.__init__ import ABFS_ROOT, normpath as abfs_normpath from desktop.lib.fs.ozone import OFS_ROOT, normpath as ofs_normpath +from desktop.lib.fs.gc import GS_ROOT, normpath as gs_normpath from desktop.lib import i18n from hadoop.fs import normpath from useradmin.models import User, Group @@ -79,6 +80,8 @@ def clean(self, value): value = '' elif value.lower().startswith(S3A_ROOT): cleaned_path = s3_normpath(cleaned_path) + elif value.lower().startswith(GS_ROOT): + cleaned_path = gs_normpath(cleaned_path) elif value.lower().startswith(ABFS_ROOT): cleaned_path = abfs_normpath(cleaned_path) elif value.lower().startswith(OFS_ROOT): diff --git a/apps/filebrowser/src/filebrowser/templates/fb_components.mako b/apps/filebrowser/src/filebrowser/templates/fb_components.mako index b8bbcc2bc24..f32e54887e2 100644 --- a/apps/filebrowser/src/filebrowser/templates/fb_components.mako +++ b/apps/filebrowser/src/filebrowser/templates/fb_components.mako @@ -40,6 +40,12 @@ else: ${ get_default_region() } + %elif path.lower().find('gs://') == 0: +
  • + + + +
    • %elif path.lower().find('adl:/') == 0:
  • diff --git a/apps/filebrowser/src/filebrowser/templates/listdir.mako b/apps/filebrowser/src/filebrowser/templates/listdir.mako index 82849a4d3c3..59d0eb52b7f 100644 --- a/apps/filebrowser/src/filebrowser/templates/listdir.mako +++ b/apps/filebrowser/src/filebrowser/templates/listdir.mako @@ -49,7 +49,7 @@ ${ fb_components.menubar() } %endif -
    +
    <%actionbar:render> @@ -61,7 +61,7 @@ ${ fb_components.menubar() }
    @@ -166,12 +166,15 @@ ${ fb_components.menubar() } ${_('Upload')} + + ${_('Upload')} + ${_('Upload')} - + @@ -183,11 +186,11 @@ ${ fb_components.menubar() }
    - +

    ${_('Are you sure you want to delete these buckets?')}

    ${_('Deleting a bucket will delete all of its contents and release the bucket name to be reserved by others.')}

    - +

    ${_('Are you sure you want to delete these files?')}

    @@ -461,10 +461,10 @@ else: