[gs] Add Google Storage filebrowser support in Hue (#3455)

What changes were proposed in this pull request? - Adding Google Storage Filebrowser in Hue as a separate Filebrowser piggybacking on S3 Filesystem implementation. - Some changes in boto package were also required to make the implementation work. How was this patch tested? - Manually. - Added new unit tests.
cloudera · Oct 6, 2023 · 56271e1 · 56271e1
1 parent c5374cb
commit 56271e1
Show file tree

Hide file tree

Showing 33 changed files with 1,316 additions and 45 deletions.
diff --git a/apps/filebrowser/src/filebrowser/api.py b/apps/filebrowser/src/filebrowser/api.py
@@ -21,6 +21,7 @@
 from desktop.lib import fsmanager
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.fs.ozone.ofs import get_ofs_home_directory
+from desktop.lib.fs.gc.gs import get_gs_home_directory
 
 from azure.abfs.__init__ import get_home_dir_for_abfs
 from aws.s3.s3fs import get_s3_home_directory
@@ -66,6 +67,8 @@ def get_filesystems_with_home_dirs(request): # Using as a public API only for no
       user_home_dir = request.user.get_home_directory()
     elif fs == 's3a':
       user_home_dir = get_s3_home_directory(request.user)
+    elif fs == 'gs':
+      user_home_dir = get_gs_home_directory(request.user)
     elif fs == 'abfs':
       user_home_dir = get_home_dir_for_abfs(request.user)
     elif fs == 'ofs':

diff --git a/apps/filebrowser/src/filebrowser/forms.py b/apps/filebrowser/src/filebrowser/forms.py
@@ -30,6 +30,7 @@
 from aws.s3 import S3A_ROOT, normpath as s3_normpath
 from azure.abfs.__init__ import ABFS_ROOT, normpath as abfs_normpath
 from desktop.lib.fs.ozone import OFS_ROOT, normpath as ofs_normpath
+from desktop.lib.fs.gc import GS_ROOT, normpath as gs_normpath
 from desktop.lib import i18n
 from hadoop.fs import normpath
 from useradmin.models import User, Group
@@ -79,6 +80,8 @@ def clean(self, value):
       value = ''
     elif value.lower().startswith(S3A_ROOT):
       cleaned_path = s3_normpath(cleaned_path)
+    elif value.lower().startswith(GS_ROOT):
+      cleaned_path = gs_normpath(cleaned_path)
     elif value.lower().startswith(ABFS_ROOT):
       cleaned_path = abfs_normpath(cleaned_path)
     elif value.lower().startswith(OFS_ROOT):

diff --git a/apps/filebrowser/src/filebrowser/templates/fb_components.mako b/apps/filebrowser/src/filebrowser/templates/fb_components.mako
@@ -40,6 +40,12 @@ else:
               <i class="fa fa-fw fa-cubes"></i> ${ get_default_region() }
             </span>
           </li>
+        %elif path.lower().find('gs://') == 0:
+          <li style="padding-top: 12px">
+            <span class="breadcrumb-link homeLink">
+              <svg class="hi"><use xlink:href='#hi-adls'></use></svg>
+            </span>
+          </li>
         %elif path.lower().find('adl:/') == 0:
           <li style="padding-top: 12px">
             <span class="breadcrumb-link homeLink">

diff --git a/apps/filebrowser/src/filebrowser/templates/listdir.mako b/apps/filebrowser/src/filebrowser/templates/listdir.mako
@@ -49,7 +49,7 @@ ${ fb_components.menubar() }
   %endif
 </style>
 
-<div id="${ path.startswith('s3a://') and 'filebrowser_s3Components' or path.startswith('abfs://') and 'filebrowser_abfsComponents'  or path.startswith('ofs://') and 'filebrowser_ofsComponents'or 'filebrowserComponents' }" class="container-fluid filebrowser" style="min-height: calc(100vh - 130px);">
+<div id="${ path.startswith('s3a://') and 'filebrowser_s3Components' or path.startswith('gs://') and 'filebrowser_gsComponents' or path.startswith('abfs://') and 'filebrowser_abfsComponents'  or path.startswith('ofs://') and 'filebrowser_ofsComponents'or 'filebrowserComponents' }" class="container-fluid filebrowser" style="min-height: calc(100vh - 130px);">
   <div class="card card-small">
     <div class="actionbar">
     <%actionbar:render>
@@ -61,7 +61,7 @@ ${ fb_components.menubar() }
         <div class="btn-toolbar" style="display: inline; vertical-align: middle">
           <div id="ch-dropdown" class="btn-group" style="vertical-align: middle">
             <button class="btn dropdown-toggle" title="${_('Actions')}" data-toggle="dropdown"
-            data-bind="visible: !inTrash(), enable: selectedFiles().length > 0 && ((!isS3() && !isABFS() && !isOFS()) || (isS3() && !isS3Root()) || (isABFS() && !isABFSRoot()) || (isOFS() && !(isOFSRoot() || isOFSServiceID() || isOFSVol())))">
+            data-bind="visible: !inTrash(), enable: selectedFiles().length > 0 && ((!isS3() && !isGS() && !isABFS() && !isOFS()) || (isS3() && !isS3Root()) || (isGS() && !isGSRoot()) || (isABFS() && !isABFSRoot()) || (isOFS() && !(isOFSRoot() || isOFSServiceID() || isOFSVol())))">
               <i class="fa fa-cog"></i> ${_('Actions')}
               <span class="caret" style="line-height: 15px"></span>
             </button>
@@ -166,12 +166,15 @@ ${ fb_components.menubar() }
           <!-- ko if: isS3 -->
             <a class="btn fileToolbarBtn" title="${_('Upload files')}" data-bind="visible: !inTrash(), css: {'disabled': isS3Root()}, click: function(){ if (!isS3Root()) { uploadFile() }}"><i class="fa fa-arrow-circle-o-up"></i> ${_('Upload')}</a>
           <!-- /ko -->
+          <!-- ko if: isGS -->
+            <a class="btn fileToolbarBtn" title="${_('Upload files')}" data-bind="visible: !inTrash(), css: {'disabled': isGSRoot()}, click: function(){ if (!isGSRoot()) { uploadFile() }}"><i class="fa fa-arrow-circle-o-up"></i> ${_('Upload')}</a>
+          <!-- /ko -->
           <!-- ko if: isABFS -->
             <a class="btn fileToolbarBtn" title="${_('Upload files')}" data-bind="visible: !inTrash(), css: {'disabled': isABFSRoot()}, click: function(){ if (!isABFSRoot()) { uploadFile() }}"><i class="fa fa-arrow-circle-o-up"></i> ${_('Upload')}</a>
           <!-- /ko -->
-          <!-- ko ifnot: isS3() || isABFS() -->
+          <!-- ko ifnot: isS3() || isGS() || isABFS() -->
           <div id="upload-dropdown" class="btn-group" style="vertical-align: middle">
-            <a data-hue-analytics="filebrowser:upload-btn-click" href="javascript: void(0)" class="btn upload-link dropdown-toggle" title="${_('Upload')}" data-bind="click: uploadFile, visible: !inTrash(), css: {'disabled': isS3() && isS3Root() || isABFS() && isABFSRoot() || (isOFS() && (isOFSRoot() || isOFSServiceID() || isOFSVol()))}">
+            <a data-hue-analytics="filebrowser:upload-btn-click" href="javascript: void(0)" class="btn upload-link dropdown-toggle" title="${_('Upload')}" data-bind="click: uploadFile, visible: !inTrash(), css: {'disabled': (isOFS() && (isOFSRoot() || isOFSServiceID() || isOFSVol()))}">
               <i class="fa fa-arrow-circle-o-up"></i> ${_('Upload')}
             </a>
           </div>
@@ -183,11 +186,11 @@ ${ fb_components.menubar() }
               <span class="caret"></span>
             </a>
             <ul class="dropdown-menu pull-right" style="top: auto">
-              <li data-bind="visible: !isS3() && !isABFS() && !isOFS() || isS3() && !isS3Root() || isABFS() && !isABFSRoot() || isOFS() && !isOFSServiceID() && !isOFSVol()"><a data-hue-analytics="filebrowser:new-file-btn-click" href="javascript: void(0)" class="create-file-link" title="${_('File')}"><i class="fa fa-file-o"></i> ${_('File')}</a></li>
+              <li data-bind="visible: !isS3() && !isGS() && !isABFS() && !isOFS() || isS3() && !isS3Root() || isGS() && !isGSRoot() || isABFS() && !isABFSRoot() || isOFS() && !isOFSServiceID() && !isOFSVol()"><a data-hue-analytics="filebrowser:new-file-btn-click" href="javascript: void(0)" class="create-file-link" title="${_('File')}"><i class="fa fa-file-o"></i> ${_('File')}</a></li>
               <li><a href="javascript: void(0)" class="create-directory-link" title="${_('Directory')}">
                 <i class="fa fa-folder"></i>
-                <span data-bind="visible: !isS3() && !isABFS() && !isOFS() || isS3() && !isS3Root() || isABFS() && !isABFSRoot() || isOFS() && !isOFSServiceID() && !isOFSVol()">${_('Directory')}</span>
-                <span data-bind="visible: (isS3() && isS3Root()) || (isOFS() && isOFSVol())">${_('Bucket')}</span>
+                <span data-bind="visible: !isS3() && !isGS() && !isABFS() && !isOFS() || isS3() && !isS3Root() || isGS() && !isGSRoot() || isABFS() && !isABFSRoot() || isOFS() && !isOFSServiceID() && !isOFSVol()">${_('Directory')}</span>
+                <span data-bind="visible: (isS3() && isS3Root()) || (isGS() && isGSRoot()) || (isOFS() && isOFSVol())">${_('Bucket')}</span>
                 <span data-bind="visible: isABFS() && isABFSRoot()">${_('File System')}</span>
                 <span data-bind="visible: isOFS() && isOFSServiceID()">${_('Volume')}</span>
               </a></li>