Adding new chunking-fileuploader template

(cherry picked from commit 517072f)
Change-Id: I9f80d24959a725c8a4765de7759b8114be808460

pop up modal close and upload job progress UI changes

(cherry picked from commit f2fb733)

Fix - drag and drop functionality - the drop zone will automatically hide when the file is moved out of the dropzone area ,
fix file upload popup css styling,
name of the file uploading will be displayed instead of static text failed.

(cherry picked from commit 7d9737b)

chunksize and max number of concurrent connections config support

(cherry picked from commit 2cf8b98)

Flag to enable new file uploader

(cherry picked from commit 4df96aa)
Change-Id: Id1eae6e920d42668858eb2b3586c5b492195e482

apps/filebrowser/src/filebrowser/conf.py

@@ -40,6 +40,18 @@
   default=None,
   type=str)
 
+FILE_UPLOAD_CHUNK_SIZE = Config(
+  key="file_upload_chunk_size",
+  default=5242880,
+  type=int,
+  help=_('Configure chunk size of the chunked file uploader. Default chunk size is set to 5MB.'))
+
+CONCURRENT_MAX_CONNECTIONS = Config(
+  key="concurrent_max_connections",
+  default=5,
+  type=int,
+  help=_('Configure the maximum number of concurrent connections(chunks) for file uploads using the chunked file uploader.'))
+
 def get_desktop_enable_download():
   """Get desktop enable_download default"""
   return ENABLE_DOWNLOAD.get()

apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -22,7 +22,7 @@ from django.template.defaultfilters import urlencode, stringformat, filesizeform
 from desktop.lib.django_util import reverse_with_get, extract_field_data
 from django.utils.encoding import smart_str
 
-from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE
+from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, FILE_UPLOAD_CHUNK_SIZE, CONCURRENT_MAX_CONNECTIONS
 
 if sys.version_info[0] > 2:
   from django.utils.translation import gettext as _
@@ -698,6 +698,31 @@ else:
 
   <script src="${ static('desktop/ext/js/datatables-paging-0.1.js') }" type="text/javascript" charset="utf-8"></script>
 
+  <script type="text/template" id="qq-template">
+    <div class="qq-uploader-selector" style="margin-left: 10px">
+        <div class="qq-upload-drop-area-selector" qq-hide-dropzone><span>${_('Drop the files here to upload')}</span></div>
+        <div class="qq-upload-button-selector qq-no-float">${_('Select files')}</div> &nbsp;
+        <span class="muted">${_('or drag and drop them here')}</span>
+
+        <ul class="qq-upload-list-selector qq-upload-files unstyled qq-no-float" style="margin-right: 0;">
+            <li>
+                <span class="qq-upload-spinner-selector hide" style="display:none"></span>
+                <div class="progress-row dz-processing">
+                    <span class="break-word qq-upload-file-selector"></span>
+                    <div class="pull-right">
+                        <span class="qq-upload-file-selector" style="display:block"></span>
+                        <span class="muted qq-upload-size-selector"></span>&nbsp;&nbsp;
+                        <a href="#" title="${_('Cancel')}" class="complex-layout"><i class="fa fa-fw fa-times qq-upload-cancel-selector"></i></a>
+                        <span class="qq-upload-done-selector" style="display:none"><i class="fa fa-fw fa-check muted"></i></span>
+                        <span class="qq-upload-failed-text">${_('Failed')}</span>
+                    </div>
+                    <div class="progress-row-bar" style="width: 0%;"></div>
+                </div>
+            </li>
+        </ul>
+    </div>
+  </script>
+
 
   <script>
     var _dragged;
@@ -2021,63 +2046,140 @@ else:
       };
 
       self.uploadFile = (function () {
-        self.pendingUploads(0);
-        var action = "/filebrowser/upload/file";
-        var uploader = new qq.FileUploader({
-          element: document.getElementById("fileUploader"),
-          action: action,
-          template: '<div class="qq-uploader" style="margin-left: 10px">' +
-          '<div class="qq-upload-drop-area"><span>${_('Drop the files here to upload')}</span></div>' +
-          '<div class="qq-upload-button qq-no-float">${_('Select files')}</div> &nbsp; <span class="muted">${_('or drag and drop them here')}</span>' +
-          '<ul class="qq-upload-list qq-upload-files unstyled qq-no-float" style="margin-right: 0;"></ul>' +
-          '</div>',
-          fileTemplate: '<li><span class="qq-upload-file-extended" style="display:none"></span><span class="qq-upload-spinner hide" style="display:none"></span>' +
-          '<div class="progress-row dz-processing">' +
-          '<span class="break-word qq-upload-file"></span>' +
-          '<div class="pull-right">' +
-          '<span class="muted qq-upload-size"></span>&nbsp;&nbsp;' +
-          '<a href="#" title="${_('Cancel')}" class="complex-layout"><i class="fa fa-fw fa-times qq-upload-cancel"></i></a>' +
-          '<span class="qq-upload-done" style="display:none"><i class="fa fa-fw fa-check muted"></i></span>' +
-          '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
-          '</div>' +
-          '<div class="progress-row-bar" style="width: 0%;"></div>' +
-          '</div></li>',
-          params: {
-            dest: self.currentPath(),
-            fileFieldLabel: "hdfs_file"
-          },
-          onProgress: function (id, fileName, loaded, total) {
-            $('.qq-upload-files').find('li').each(function(){
-              var listItem = $(this);
-              if (listItem.find('.qq-upload-file-extended').text() == fileName){
-                listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
-              }
+          var uploader;  
+          if (window.getLastKnownConfig().hue_config.enable_chunked_file_uploader) {
+            self.pendingUploads(0);
+            var action = "/filebrowser/upload/chunks/";
+            uploader = new qq.FileUploader({
+              element: document.getElementById("fileUploader"),
+              request: {
+                  endpoint: action,
+                  paramsInBody: false,
+                  params: {
+                      dest: self.currentPath(),
+                      inputName: "hdfs_file"
+                  }
+              },
+              maxConnections: window.CONCURRENT_MAX_CONNECTIONS || 5,
+              chunking: {
+                  enabled: true,
+                  concurrent: {
+                      enabled: true
+                  },
+                  partSize: window.FILE_UPLOAD_CHUNK_SIZE || 5242880,
+                  success: {
+                      endpoint: "/filebrowser/upload/complete/"
+                  },
+                  paramNames: {
+                      partIndex: "qqpartindex",
+                      partByteOffset: "qqpartbyteoffset",
+                      chunkSize: "qqchunksize",
+                      totalFileSize: "qqtotalfilesize",
+                      totalParts: "qqtotalparts"
+                  }
+              },
+
+              template: 'qq-template',
+              callbacks: {
+                  onProgress: function (id, fileName, loaded, total) {
+                  console.log(loaded);
+                  $('.qq-upload-files').find('li').each(function(){
+                    var listItem = $(this);
+                    if (listItem.find('.qq-upload-file-selector').text() == fileName){
+                      listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
+                    }
+                  });
+                  },
+                  onComplete: function (id, fileName, response) {
+                    self.pendingUploads(self.pendingUploads() - 1);
+                    if (response.status != 0) {
+                      $(document).trigger('error', "${ _('Error: ') }" + response.data);
+                    }
+                    else {
+                      $(document).trigger('info', response.path + "${ _(' uploaded successfully.') }");
+                      self.filesToHighlight.push(response.path);
+                    }
+                    if (self.pendingUploads() == 0) {
+                      $('#uploadFileModal').modal('hide');
+                      self.retrieveData(true);
+                    }
+                  },
+
+                  onAllComplete: function(succeeded, failed){
+                    $('#uploadFileModal').modal('hide');
+                  },
+                  onSubmit: function (id, fileName, responseJSON) {
+                    var newPath = "/filebrowser/upload/chunks/file?dest=" + encodeURIComponent(self.currentPath().normalize('NFC'));
+                    this.setEndpoint(newPath);
+                    self.pendingUploads(self.pendingUploads() + 1);
+                  },
+                  onCancel: function (id, fileName) {
+                    self.pendingUploads(self.pendingUploads() - 1);
+                  }
+              },
+
+              debug: false
             });
-          },
-          onComplete: function (id, fileName, response) {
-            self.pendingUploads(self.pendingUploads() - 1);
-            if (response.status != 0) {
-              huePubSub.publish('hue.global.error', {message: "${ _('Error: ') }" + response.data});
-            }
-            else {
-              $(document).trigger('info', response.path + "${ _(' uploaded successfully.') }");
-              self.filesToHighlight.push(response.path);
-            }
-            if (self.pendingUploads() == 0) {
-              $('#uploadFileModal').modal('hide');
-              self.retrieveData(true);
-            }
-          },
-          onSubmit: function (id, fileName, responseJSON) {
-            self.pendingUploads(self.pendingUploads() + 1);
-          },
-          onCancel: function (id, fileName) {
-            self.pendingUploads(self.pendingUploads() - 1);
-          },
-          debug: false
-        });
+          }
+          else {
+            self.pendingUploads(0);
+            var action = "/filebrowser/upload/file";
+            uploader = new fileuploader.FileUploader({
+              element: document.getElementById("fileUploader"),
+              action: action,
+              template: '<div class="qq-uploader" style="margin-left: 10px">' +
+              '<div class="qq-upload-drop-area"><span>${_('Drop the files here to upload')}</span></div>' +
+              '<div class="qq-upload-button qq-no-float">${_('Select files')}</div> &nbsp; <span class="muted">${_('or drag and drop them here')}</span>' +
+              '<ul class="qq-upload-list qq-upload-files unstyled qq-no-float" style="margin-right: 0;"></ul>' +
+              '</div>',
+              fileTemplate: '<li><span class="qq-upload-file-extended" style="display:none"></span><span class="qq-upload-spinner hide" style="display:none"></span>' +
+              '<div class="progress-row dz-processing">' +
+              '<span class="break-word qq-upload-file"></span>' +
+              '<div class="pull-right">' +
+              '<span class="muted qq-upload-size"></span>&nbsp;&nbsp;' +
+              '<a href="#" title="${_('Cancel')}" class="complex-layout"><i class="fa fa-fw fa-times qq-upload-cancel"></i></a>' +
+              '<span class="qq-upload-done" style="display:none"><i class="fa fa-fw fa-check muted"></i></span>' +
+              '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
+              '</div>' +
+              '<div class="progress-row-bar" style="width: 0%;"></div>' +
+              '</div></li>',
+              params: {
+                dest: self.currentPath(),
+                fileFieldLabel: "hdfs_file"
+              },
+              onProgress: function (id, fileName, loaded, total) {
+                $('.qq-upload-files').find('li').each(function(){
+                  var listItem = $(this);
+                  if (listItem.find('.qq-upload-file-extended').text() == fileName){
+                    listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
+                  }
+                });
+              },
+              onComplete: function (id, fileName, response) {
+                self.pendingUploads(self.pendingUploads() - 1);
+                if (response.status != 0) {
+                  $(document).trigger('error', "${ _('Error: ') }" + response.data);
+                }
+                else {
+                  $(document).trigger('info', response.path + "${ _(' uploaded successfully.') }");
+                  self.filesToHighlight.push(response.path);
+                }
+                if (self.pendingUploads() == 0) {
+                  $('#uploadFileModal').modal('hide');
+                  self.retrieveData(true);
+                }
+              },
+              onSubmit: function (id, fileName, responseJSON) {
+                self.pendingUploads(self.pendingUploads() + 1);
+              },
+              onCancel: function (id, fileName) {
+                self.pendingUploads(self.pendingUploads() - 1);
+              },
+              debug: false
+            });
+          }
 
-        $("#fileUploader").on('fb:updatePath', function (e, options) {
+          $("#fileUploader").on('fb:updatePath', function (e, options) {
           const uploadingToOzone = self.currentPath().startsWith("ofs://");
           const ozoneSizeLimit = Math.min(
             ...[UPLOAD_CHUNK_SIZE, MAX_FILE_SIZE_UPLOAD_LIMIT].filter(Number.isFinite)
@@ -2641,7 +2743,9 @@ else:
           _dropzone.enable();
         }
         $(".qq-upload-list").empty();
+        $(".qq-upload-list-selector").empty();
         $(".qq-upload-drop-area").hide();
+        $(".qq-upload-drop-area-selector").hide();
       });
     });
   </script>

desktop/core/src/desktop/api2.py

@@ -46,8 +46,8 @@
 from beeswax.models import Namespace
 from desktop import appmanager
 from desktop.auth.backend import is_admin
-from desktop.conf import ENABLE_CONNECTORS, ENABLE_GIST_PREVIEW, CUSTOM, get_clusters, ENABLE_SHARING
-from desktop.conf import ENABLE_NEW_STORAGE_BROWSER
+from desktop.conf import ENABLE_CONNECTORS, ENABLE_GIST_PREVIEW, CUSTOM, get_clusters, IS_K8S_ONLY, ENABLE_SHARING
+from desktop.conf import ENABLE_NEW_STORAGE_BROWSER, ENABLE_CHUNKED_FILE_UPLOADER
 from desktop.lib.conf import BoundContainer, GLOBAL_CONFIG, is_anonymous
 from desktop.lib.django_util import JsonResponse, login_notrequired, render
 from desktop.lib.exceptions_renderable import PopupException
@@ -100,6 +100,7 @@ def get_config(request):
   config['hue_config']['is_admin'] = is_admin(request.user)
   config['hue_config']['is_yarn_enabled'] = is_yarn()
   config['hue_config']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
+  config['hue_config']['enable_chunked_file_uploader'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
   config['clusters'] = list(get_clusters(request.user).values())
   config['documents'] = {
     'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct())

desktop/core/src/desktop/conf.py

@@ -1818,6 +1818,16 @@ def get_instrumentation_default():
   default=False
 )
 
+def is_chunked_fileuploader_enabled():
+  return ENABLE_CHUNKED_FILE_UPLOADER.get();
+
+ENABLE_CHUNKED_FILE_UPLOADER = Config(
+  key="enable_chunked_file_uploader",
+  help=_("Enable new chunked file uploader."),
+  type=coerce_bool,
+  default=False
+)
+
 USE_NEW_EDITOR = Config( # To remove in Hue 4
   key='',
   default=True,

desktop/core/src/desktop/js/ext/fileuploader.custom.js

@@ -1259,7 +1259,7 @@ qq.extend(qq.UploadHandlerXhr.prototype, {
             }
         };
 
-        var formData = new FormData();        
+        var formData = new FormData();
         formData.append(params.fileFieldLabel, file, file.name.normalize('NFC'));
         formData.append('dest', params.dest);
 
@@ -1313,4 +1313,5 @@ qq.extend(qq.UploadHandlerXhr.prototype, {
     }
 });
 
-export default qq;
+let fileuploader = qq;
+export default fileuploader;