Skip to content

Commit

Permalink
Flag to enable new file uploader
Browse files Browse the repository at this point in the history 
(cherry picked from commit 4df96aa)
Change-Id: Id1eae6e920d42668858eb2b3586c5b492195e482
  • Loading branch information
Athithyaa Selvam committed Nov 1, 2023
1 parent aeb0b41 commit 92f5e86
Show file tree
Hide file tree
Showing 9 changed files with 9,184 additions and 7,780 deletions.
6 changes: 3 additions & 3 deletions apps/filebrowser/src/filebrowser/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@

FILE_UPLOAD_CHUNK_SIZE = Config(
key="file_upload_chunk_size",
default=5000000,
default=5242880,
type=int,
help=_('flag to change the chunk size of file upload'))
help=_('Configure chunk size of the chunked file uploader. Default chunk size is set to 5MB.'))

CONCURRENT_MAX_CONNECTIONS = Config(
key="concurrent_max_connections",
default=5,
type=int,
help=_('flag to set the maximum number of conncurrent connections to upload file'))
help=_('Configure the maximum number of concurrent connections(chunks) for file uploads using the chunked file uploader.'))

def get_desktop_enable_download():
"""Get desktop enable_download default"""
Expand Down
161 changes: 111 additions & 50 deletions apps/filebrowser/src/filebrowser/templates/listdir_components.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2046,48 +2046,114 @@ else:
};
self.uploadFile = (function () {
self.pendingUploads(0);
var action = "/filebrowser/uploadchunks/";
// var dest = "?%2Fuser%2Fadmin"
var uploader = new qq.FileUploader({
element: document.getElementById("fileUploader"),
request: {
endpoint: action,
paramsInBody: false,
params: {
dest: self.currentPath(),
inputName: "hdfs_file"
}
},
maxConnections: window.CONCURRENT_MAX_CONNECTIONS || 5,
chunking: {
enabled: true,
concurrent: {
enabled: true
var uploader;
if (window.getLastKnownConfig().hue_config.enable_chunked_file_uploader) {
self.pendingUploads(0);
var action = "/filebrowser/upload/chunks/";
uploader = new qq.FileUploader({
element: document.getElementById("fileUploader"),
request: {
endpoint: action,
paramsInBody: false,
params: {
dest: self.currentPath(),
inputName: "hdfs_file"
}
},
partSize: window.FILE_UPLOAD_CHUNK_SIZE || 5000000,
success: {
endpoint: "/filebrowser/uploaddone/"
maxConnections: window.CONCURRENT_MAX_CONNECTIONS || 5,
chunking: {
enabled: true,
concurrent: {
enabled: true
},
partSize: window.FILE_UPLOAD_CHUNK_SIZE || 5242880,
success: {
endpoint: "/filebrowser/upload/complete/"
},
paramNames: {
partIndex: "qqpartindex",
partByteOffset: "qqpartbyteoffset",
chunkSize: "qqchunksize",
totalFileSize: "qqtotalfilesize",
totalParts: "qqtotalparts"
}
},
template: 'qq-template',
callbacks: {
onProgress: function (id, fileName, loaded, total) {
console.log(loaded);
$('.qq-upload-files').find('li').each(function(){
var listItem = $(this);
if (listItem.find('.qq-upload-file-selector').text() == fileName){
listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
}
});
},
onComplete: function (id, fileName, response) {
self.pendingUploads(self.pendingUploads() - 1);
if (response.status != 0) {
$(document).trigger('error', "${ _('Error: ') }" + response.data);
}
else {
$(document).trigger('info', response.path + "${ _(' uploaded successfully.') }");
self.filesToHighlight.push(response.path);
}
if (self.pendingUploads() == 0) {
$('#uploadFileModal').modal('hide');
self.retrieveData(true);
}
},
onAllComplete: function(succeeded, failed){
$('#uploadFileModal').modal('hide');
},
onSubmit: function (id, fileName, responseJSON) {
var newPath = "/filebrowser/upload/chunks/file?dest=" + encodeURIComponent(self.currentPath().normalize('NFC'));
this.setEndpoint(newPath);
self.pendingUploads(self.pendingUploads() + 1);
},
onCancel: function (id, fileName) {
self.pendingUploads(self.pendingUploads() - 1);
}
},
paramNames: {
partIndex: "qqpartindex",
partByteOffset: "qqpartbyteoffset",
chunkSize: "qqchunksize",
totalFileSize: "qqtotalfilesize",
totalParts: "qqtotalparts"
}
},
template: 'qq-template',
callbacks: {
debug: false
});
}
else {
self.pendingUploads(0);
var action = "/filebrowser/upload/file";
uploader = new fileuploader.FileUploader({
element: document.getElementById("fileUploader"),
action: action,
template: '<div class="qq-uploader" style="margin-left: 10px">' +
'<div class="qq-upload-drop-area"><span>${_('Drop the files here to upload')}</span></div>' +
'<div class="qq-upload-button qq-no-float">${_('Select files')}</div> &nbsp; <span class="muted">${_('or drag and drop them here')}</span>' +
'<ul class="qq-upload-list qq-upload-files unstyled qq-no-float" style="margin-right: 0;"></ul>' +
'</div>',
fileTemplate: '<li><span class="qq-upload-file-extended" style="display:none"></span><span class="qq-upload-spinner hide" style="display:none"></span>' +
'<div class="progress-row dz-processing">' +
'<span class="break-word qq-upload-file"></span>' +
'<div class="pull-right">' +
'<span class="muted qq-upload-size"></span>&nbsp;&nbsp;' +
'<a href="#" title="${_('Cancel')}" class="complex-layout"><i class="fa fa-fw fa-times qq-upload-cancel"></i></a>' +
'<span class="qq-upload-done" style="display:none"><i class="fa fa-fw fa-check muted"></i></span>' +
'<span class="qq-upload-failed-text">${_('Failed')}</span>' +
'</div>' +
'<div class="progress-row-bar" style="width: 0%;"></div>' +
'</div></li>',
params: {
dest: self.currentPath(),
fileFieldLabel: "hdfs_file"
},
onProgress: function (id, fileName, loaded, total) {
console.log(loaded);
$('.qq-upload-files').find('li').each(function(){
var listItem = $(this);
if (listItem.find('.qq-upload-file-selector').text() == fileName){
listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
}
});
$('.qq-upload-files').find('li').each(function(){
var listItem = $(this);
if (listItem.find('.qq-upload-file-extended').text() == fileName){
listItem.find('.progress-row-bar').css('width', (loaded/total)*100 + '%');
}
});
},
onComplete: function (id, fileName, response) {
self.pendingUploads(self.pendingUploads() - 1);
Expand All @@ -2103,24 +2169,17 @@ else:
self.retrieveData(true);
}
},
onAllComplete: function(succeeded, failed){
$('#uploadFileModal').modal('hide');
},
onSubmit: function (id, fileName, responseJSON) {
var newPath = "/filebrowser/uploadchunks/file?dest=" + encodeURIComponent(self.currentPath());
this.setEndpoint(newPath);
self.pendingUploads(self.pendingUploads() + 1);
},
onCancel: function (id, fileName) {
self.pendingUploads(self.pendingUploads() - 1);
}
},
debug: false
});
},
debug: false
});
}
$("#fileUploader").on('fb:updatePath', function (e, options) {
$("#fileUploader").on('fb:updatePath', function (e, options) {
const uploadingToOzone = self.currentPath().startsWith("ofs://");
const ozoneSizeLimit = Math.min(
...[UPLOAD_CHUNK_SIZE, MAX_FILE_SIZE_UPLOAD_LIMIT].filter(Number.isFinite)
Expand Down Expand Up @@ -2683,8 +2742,10 @@ else:
if (typeof _dropzone != "undefined") {
_dropzone.enable();
}
$(".qq-upload-list").empty();
$(".qq-upload-list-selector").empty();
$(".qq-upload-drop-area").hide();
$(".qq-upload-drop-area-selector").hide();
});
});
</script>
Expand Down
5 changes: 3 additions & 2 deletions desktop/core/src/desktop/api2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@
from beeswax.models import Namespace
from desktop import appmanager
from desktop.auth.backend import is_admin
from desktop.conf import ENABLE_CONNECTORS, ENABLE_GIST_PREVIEW, CUSTOM, get_clusters, ENABLE_SHARING
from desktop.conf import ENABLE_NEW_STORAGE_BROWSER
from desktop.conf import ENABLE_CONNECTORS, ENABLE_GIST_PREVIEW, CUSTOM, get_clusters, IS_K8S_ONLY, ENABLE_SHARING
from desktop.conf import ENABLE_NEW_STORAGE_BROWSER, ENABLE_CHUNKED_FILE_UPLOADER
from desktop.lib.conf import BoundContainer, GLOBAL_CONFIG, is_anonymous
from desktop.lib.django_util import JsonResponse, login_notrequired, render
from desktop.lib.exceptions_renderable import PopupException
Expand Down Expand Up @@ -100,6 +100,7 @@ def get_config(request):
config['hue_config']['is_admin'] = is_admin(request.user)
config['hue_config']['is_yarn_enabled'] = is_yarn()
config['hue_config']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
config['hue_config']['enable_chunked_file_uploader'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
config['clusters'] = list(get_clusters(request.user).values())
config['documents'] = {
'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct())
Expand Down
10 changes: 10 additions & 0 deletions desktop/core/src/desktop/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1818,6 +1818,16 @@ def get_instrumentation_default():
default=False
)

def is_chunked_fileuploader_enabled():
return ENABLE_CHUNKED_FILE_UPLOADER.get();

ENABLE_CHUNKED_FILE_UPLOADER = Config(
key="enable_chunked_file_uploader",
help=_("Enable new chunked file uploader."),
type=coerce_bool,
default=False
)

USE_NEW_EDITOR = Config( # To remove in Hue 4
key='',
default=True,
Expand Down
Loading

0 comments on commit 92f5e86

Please sign in to comment.