diff --git a/gen3-client/g3cmd/gitversion.go b/gen3-client/g3cmd/gitversion.go index cb3a3088..c29bb19f 100644 --- a/gen3-client/g3cmd/gitversion.go +++ b/gen3-client/g3cmd/gitversion.go @@ -1,6 +1,6 @@ package g3cmd -var ( +const ( gitcommit = "N/A" - gitversion = "2023.11" + gitversion = "N/A" ) diff --git a/gen3-client/g3cmd/upload-multiple.go b/gen3-client/g3cmd/upload-multiple.go index 3b117882..549f055d 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/gen3-client/g3cmd/upload-multiple.go @@ -79,44 +79,21 @@ func init() { log.Fatalf("Error when parsing file paths: " + err.Error()) } - for i, furObject := range furObjects { - if batch { - if len(batchFURObjects) < workers { - batchFURObjects = append(batchFURObjects, furObject) - } else { - batchUpload(gen3Interface, batchFURObjects, workers, respCh, errCh, bucketName) - batchFURObjects = make([]commonUtils.FileUploadRequestObject, 0) - batchFURObjects = append(batchFURObjects, furObject) - } - if i == len(furObjects)-1 { // upload remainders - batchUpload(gen3Interface, batchFURObjects, workers, respCh, errCh, bucketName) - } + filePaths := make([]string, 0) + for _, object := range objects { + var filePath string + var err error + + if object.Filename != "" { + // conform to fence naming convention + filePath, err = getFullFilePath(uploadPath, object.Filename) } else { - file, err := os.Open(furObject.FilePath) - if err != nil { - log.Println("File open error: " + err.Error()) - logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true) - logs.IncrementScore(logs.ScoreBoardLen - 1) - continue - } - defer file.Close() - - furObject, err := GenerateUploadRequest(gen3Interface, furObject, file) - if err != nil { - file.Close() - logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true) - logs.IncrementScore(logs.ScoreBoardLen - 1) - log.Printf("Error occurred during request generation: %s", err.Error()) - continue - } - err = uploadFile(furObject, 0) - if err != nil { - log.Println(err.Error()) - logs.IncrementScore(logs.ScoreBoardLen - 1) - } else { - logs.IncrementScore(0) - } - file.Close() + // Otherwise, here we are assuming the local filename will be the same as GUID + filePath, err = getFullFilePath(uploadPath, object.ObjectID) + } + if err != nil { + log.Println(err.Error()) + continue } filePaths = append(filePaths, filePath) } @@ -167,7 +144,9 @@ func init() { uploadMultipleCmd.MarkFlagRequired("upload-path") //nolint:errcheck uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload in parallel") uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") - uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded") + uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") + uploadMultipleCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload when possible (file size >= 5MB)") + uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", false, "Include subdirectory names in file name") RootCmd.AddCommand(uploadMultipleCmd) } diff --git a/gen3-client/g3cmd/upload-single.go b/gen3-client/g3cmd/upload-single.go index 3b1bf552..1e85edd1 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/gen3-client/g3cmd/upload-single.go @@ -95,6 +95,6 @@ func init() { uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file") uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded") + uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") RootCmd.AddCommand(uploadSingleCmd) } diff --git a/gen3-client/g3cmd/upload.go b/gen3-client/g3cmd/upload.go index da3a396c..3e290833 100644 --- a/gen3-client/g3cmd/upload.go +++ b/gen3-client/g3cmd/upload.go @@ -154,26 +154,7 @@ func init() { if len(multipartFilePaths) > 0 { // NOTE(@mpingram) - For the moment Shepherd doesn't support multipart uploads. // Throw an error if Shepherd is enabled and user attempts to multipart upload. - profileConfig := conf.ParseConfig(profile) - if profileConfig.UseShepherd == "true" || - profileConfig.UseShepherd == "" && commonUtils.DefaultUseShepherd == true { - log.Fatalf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) - } - log.Println("Multipart uploading....") - for _, filePath := range multipartFilePaths { - fileInfo, err := ProcessFilename(uploadPath, filePath, includeSubDirName, false) - if err != nil { - logs.AddToFailedLog(filePath, filepath.Base(filePath), commonUtils.FileMetadata{}, "", 0, false, true) - log.Println("Process filename error for file: " + err.Error()) - continue - } - err = multipartUpload(gen3Interface, fileInfo, 0, bucketName) - if err != nil { - log.Println(err.Error()) - } else { - logs.IncrementScore(0) - } - } + processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) } if !logs.IsFailedLogMapEmpty() { @@ -193,6 +174,6 @@ func init() { uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", false, "Include subdirectory names in file name") uploadCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload if possible") uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file") - uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded") + uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") RootCmd.AddCommand(uploadCmd) } diff --git a/gen3-client/g3cmd/utils.go b/gen3-client/g3cmd/utils.go index 540896f6..8dd1e3a9 100644 --- a/gen3-client/g3cmd/utils.go +++ b/gen3-client/g3cmd/utils.go @@ -537,37 +537,6 @@ func getFullFilePath(filePath string, filename string) (string, error) { } } -func validateObject(objects []ManifestObject, uploadPath string) []commonUtils.FileUploadRequestObject { - furObjects := make([]commonUtils.FileUploadRequestObject, 0) - for _, object := range objects { - guid := object.ObjectID - var fileName = "" - - if object.Filename != "" { - // conform to fence naming convention - fileName = object.Filename - } else { - // Otherwise, here we are assuming the local filename will be the same as GUID - fileName = object.ObjectID - } - - filePath, err := getFullFilePath(uploadPath, fileName) - if err != nil { - log.Println(err.Error()) - continue - } - - if _, err := os.Stat(filePath); os.IsNotExist(err) { - log.Printf("The file you specified \"%s\" does not exist locally.\n", filePath) - continue - } - - furObject := commonUtils.FileUploadRequestObject{FilePath: filePath, Filename: fileName, GUID: guid} - furObjects = append(furObjects, furObject) - } - return furObjects -} - func uploadFile(furObject commonUtils.FileUploadRequestObject, retryCount int) error { log.Println("Uploading data ...") furObject.Bar.Start()