diff --git a/Dockerfile b/Dockerfile index 8db04d5..cc7000c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV CGO_ENABLED=0 ENV GOOS=linux ENV GOARCH=amd64 -WORKDIR $GOPATH/src/github.com/uc-cdis/gen3-client/ +WORKDIR $GOPATH/src/github.com/calypr/data-client/ COPY go.mod . COPY go.sum . @@ -20,10 +20,10 @@ RUN COMMIT=$(git rev-parse HEAD); \ 'const ('\ ' gitcommit="'"${COMMIT}"'"'\ ' gitversion="'"${VERSION}"'"'\ - ')' > gen3-client/g3cmd/gitversion.go \ - && go build -o /gen3-client + ')' > data-client/g3cmd/gitversion.go \ + && go build -o /data-client FROM scratch COPY --from=build-deps /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=build-deps /gen3-client /gen3-client -CMD ["/gen3-client"] +COPY --from=build-deps /data-client /data-client +CMD ["/data-client"] diff --git a/README.md b/README.md index 48586e9..64ec2f3 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,24 @@ -# gen3-client +# data-client [![Build Status](https://travis-ci.org/uc-cdis/cdis-data-client.svg?branch=master)](https://travis-ci.org/uc-cdis/cdis-data-client) [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/uc-cdis/cdis-data-client?sort=semver)](https://github.com/uc-cdis/cdis-data-client/releases) -`gen3-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. +`data-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. -Read more about what it does and how to use it in the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/). +Read more about what it does and how to use it in the `data-client` [user guide](https://gen3.org/resources/user/data-client/). -`gen3-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra). +`data-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra). ## Installation -(The following instruction is for compiling and installing the `gen3-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases)) +(The following instruction is for compiling and installing the `data-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases)) First, [install Go and the Go tools](https://golang.org/doc/install) if you have not already done so. [Set up your workspace and your GOPATH.](https://golang.org/doc/code.html) Then: ``` -go get -d github.com/uc-cdis/gen3-client +go get -d github.com/calypr/data-client go install ``` @@ -30,43 +30,43 @@ go install mkdir -p $GOPATH/src/github.com/uc-cdis cd $GOPATH/src/github.com/uc-cdis git clone git@github.com:uc-cdis/cdis-data-client.git -mv cdis-data-client gen3-client -cd gen3-client +mv cdis-data-client data-client +cd data-client go get -d ./... go install . ``` -Now you should have `gen3-client` successfully installed. For a comprehensive instruction on how to configure and use `gen3-client` for uploading / downloading object files, please refer to the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/). +Now you should have `data-client` successfully installed. For a comprehensive instruction on how to configure and use `data-client` for uploading / downloading object files, please refer to the `data-client` [user guide](https://gen3.org/resources/user/data-client/). ## Enabling New Gen3 Object Management API Some Gen3 data commons support uploading files through the new Gen3 Object Management API. > NOTE: The service powering this API is sometimes referred to as our object "Shepherd" -To enable gen3-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `gen3-client configure`, e.g.: +To enable data-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `data-client configure`, e.g.: ``` -$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true +$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true ``` -If this flag is set, the gen3-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure. +If this flag is set, the data-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure. >You may also need to configure the version of the Gen3 Object Management API that the client will interact with. This is set to a default of Gen3 Object Management API `v2.0.0`, but can ->be raised or lowered by passing the `min-shepherd-version` flag to `gen3-client configure`, e.g.: +>be raised or lowered by passing the `min-shepherd-version` flag to `data-client configure`, e.g.: >``` ->$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0 +>$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0 >``` ### Uploading Additional File Object Metadata to Gen3 Object Management API -The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files. +The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files. > WARNING: Additional File Object Metadata is exposed publically and thus should not be controlled/sensitive data -You can upload file metadata using the `gen3-client upload` command with the `--metadata` flag. E.g.: +You can upload file metadata using the `data-client upload` command with the `--metadata` flag. E.g.: ``` -gen3-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata +data-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata ``` -This will tell `gen3-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`. +This will tell `data-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`. A metadata file should be located in the same folder as the file to be uploaded, and should be named `[filename]_metadata.json`. The metadata file should be a JSON file in the format: diff --git a/gen3-client/commonUtils/commonUtils.go b/data-client/commonUtils/commonUtils.go similarity index 93% rename from gen3-client/commonUtils/commonUtils.go rename to data-client/commonUtils/commonUtils.go index 23a88f5..2737305 100644 --- a/gen3-client/commonUtils/commonUtils.go +++ b/data-client/commonUtils/commonUtils.go @@ -17,13 +17,13 @@ import ( // DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API // endpoints if available. -// The user can override this default using the `gen3-client configure` command. +// The user can override this default using the `data-client configure` command. const DefaultUseShepherd = false // DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use. // Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is // below this number the gen3client will instead warn the user and fall back to fence/indexd. -// The user can override this default using the `gen3-client configure` command. +// The user can override this default using the `data-client configure` command. const DefaultMinShepherdVersion = "2.0.0" // ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API @@ -74,7 +74,7 @@ type FileUploadRequestObject struct { PresignedURL string Request *http.Request Bar *pb.ProgressBar - Bucket string `json:"bucket,omitempty"` + Bucket string `json:"bucket,omitempty"` } // FileDownloadResponseObject defines a object for file download @@ -106,25 +106,28 @@ type RetryObject struct { GUID string RetryCount int Multipart bool - Bucket string + Bucket string } // ParseRootPath parses dirname that has "~" in the beginning -func ParseRootPath(filePath string) string { +func ParseRootPath(filePath string) (string, error) { if filePath != "" && filePath[0] == '~' { homeDir, err := homedir.Dir() if err != nil { - log.Fatalln(err) + return "", err } - return homeDir + filePath[1:] + return homeDir + filePath[1:], nil } - return filePath + return filePath, nil } // GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning func GetAbsolutePath(filePath string) (string, error) { - fullFilePath := ParseRootPath(filePath) - fullFilePath, err := filepath.Abs(fullFilePath) + fullFilePath, err := ParseRootPath(filePath) + if err != nil { + return "", err + } + fullFilePath, err = filepath.Abs(fullFilePath) return fullFilePath, err } diff --git a/gen3-client/commonUtils/isHidden_notwindows.go b/data-client/commonUtils/isHidden_notwindows.go similarity index 100% rename from gen3-client/commonUtils/isHidden_notwindows.go rename to data-client/commonUtils/isHidden_notwindows.go diff --git a/gen3-client/commonUtils/isHidden_windows.go b/data-client/commonUtils/isHidden_windows.go similarity index 100% rename from gen3-client/commonUtils/isHidden_windows.go rename to data-client/commonUtils/isHidden_windows.go diff --git a/gen3-client/g3cmd/auth.go b/data-client/g3cmd/auth.go similarity index 90% rename from gen3-client/g3cmd/auth.go rename to data-client/g3cmd/auth.go index cd8d4a2..af47e81 100644 --- a/gen3-client/g3cmd/auth.go +++ b/data-client/g3cmd/auth.go @@ -6,7 +6,7 @@ import ( "sort" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/logs" ) func init() { @@ -15,12 +15,15 @@ func init() { Use: "auth", Short: "Return resource access privileges from profile", Long: `Gets resource access privileges for specified profile.`, - Example: `./gen3-client auth --profile=`, + Example: `./data-client auth --profile=`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + profileConfig, err := conf.ParseConfig(profile) + if err != nil { + log.Fatalf("Fatal config parse error: %s\n", err) + } host, resourceAccess, err := gen3Interface.CheckPrivileges(&profileConfig) diff --git a/gen3-client/g3cmd/configure.go b/data-client/g3cmd/configure.go similarity index 82% rename from gen3-client/g3cmd/configure.go rename to data-client/g3cmd/configure.go index 5412c33..0996121 100644 --- a/gen3-client/g3cmd/configure.go +++ b/data-client/g3cmd/configure.go @@ -2,11 +2,12 @@ package g3cmd import ( "fmt" + "log" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) var conf jwt.Configure // Why is this a global variable? @@ -21,12 +22,15 @@ func init() { Short: "Add or modify a configuration profile to your config file", Long: `Configuration file located at ~/.gen3/gen3_client_config.ini If a field is left empty, the existing value (if it exists) will remain unchanged`, - Example: `./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, + Example: `./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion) + err := jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion) + if err != nil { + log.Println(err.Error()) + } }, } diff --git a/gen3-client/g3cmd/delete.go b/data-client/g3cmd/delete.go similarity index 86% rename from gen3-client/g3cmd/delete.go rename to data-client/g3cmd/delete.go index 95d90ee..5be6795 100644 --- a/gen3-client/g3cmd/delete.go +++ b/data-client/g3cmd/delete.go @@ -13,8 +13,8 @@ var deleteCmd = &cobra.Command{ // nolint:deadcode,unused,varcheck Short: "Send DELETE HTTP Request for given URI", Long: `Deletes a given URI from the database. If no profile is specified, "default" profile is used for authentication.`, - Example: `./gen3-client delete --uri=v0/submission/bpa/test/entities/example_id - ./gen3-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, + Example: `./data-client delete --uri=v0/submission/bpa/test/entities/example_id + ./data-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, Run: func(cmd *cobra.Command, args []string) { log.Fatalf("Not supported!") // request := new(jwt.Request) diff --git a/gen3-client/g3cmd/download-multiple.go b/data-client/g3cmd/download-multiple.go similarity index 92% rename from gen3-client/g3cmd/download-multiple.go rename to data-client/g3cmd/download-multiple.go index ed1c210..0b42213 100644 --- a/gen3-client/g3cmd/download-multiple.go +++ b/data-client/g3cmd/download-multiple.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -14,8 +13,8 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" pb "gopkg.in/cheggaaa/pb.v1" "github.com/spf13/cobra" @@ -158,9 +157,9 @@ func processOriginalFilename(downloadPath string, actualFilename string) string } } -func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) { +func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) error { if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" { - log.Fatalln("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") + return fmt.Errorf("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") } if filenameFormat == "guid" || filenameFormat == "combined" { fmt.Printf("WARNING: in \"guid\" or \"combined\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) @@ -177,6 +176,7 @@ func validateFilenameFormat(downloadPath string, filenameFormat string, rename b } else { fmt.Printf("NOTICE: flag \"rename\" was set to true in \"original\" mode, duplicated files under \"%s\" will be renamed by appending a counter value to the original filenames\n", downloadPath) } + return nil } func validateLocalFileStat(downloadPath string, filename string, filesize int64, skipCompleted bool) commonUtils.FileDownloadResponseObject { @@ -284,12 +284,15 @@ func batchDownload(g3 Gen3Interface, batchFDRSlice []commonUtils.FileDownloadRes return succeeded } -func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) { +func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) error { if numParallel < 1 { - log.Fatalln("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.") + return fmt.Errorf("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.") } - downloadPath = commonUtils.ParseRootPath(downloadPath) + downloadPath, err := commonUtils.ParseRootPath(downloadPath) + if err != nil { + return fmt.Errorf("downloadFile Error: %s", err.Error()) + } if !strings.HasSuffix(downloadPath, "/") { downloadPath += "/" } @@ -298,16 +301,19 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat fmt.Println("NOTICE: flag \"rename\" only works if flag \"filename-format\" is \"original\"") rename = false } - validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt) + err = validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt) + if err != nil { + return err + } protocolText := "" if protocol != "" { protocolText = "?protocol=" + protocol } - err := os.MkdirAll(downloadPath, 0766) + err = os.MkdirAll(downloadPath, 0766) if err != nil { - log.Fatalln("Cannot create folder \"" + downloadPath + "\"") + return fmt.Errorf("Cannot create folder %s", downloadPath) } renamedFiles := make([]RenamedOrSkippedFileInfo, 0) @@ -381,6 +387,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat log.Println(err.Error()) } } + return nil } func init() { @@ -397,11 +404,15 @@ func init() { Use: "download-multiple", Short: "Download multiple of files from a specified manifest", Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, - Example: `./gen3-client download-multiple --profile= --manifest= --download-path=`, + Example: `./data-client download-multiple --profile= --manifest= --download-path=`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - profileConfig = conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } manifestPath, _ = commonUtils.GetAbsolutePath(manifestPath) manifestFile, err := os.Open(manifestPath) @@ -420,23 +431,25 @@ func init() { manifestFileReader := manifestFileBar.NewProxyReader(manifestFile) - manifestBytes, err := ioutil.ReadAll(manifestFileReader) - manifestFileBar.Finish() - + manifestBytes, err := io.ReadAll(manifestFileReader) if err != nil { - log.Printf("Failed reading manifest %s, %v\n", manifestPath, err) - log.Fatalln("A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") + log.Fatalf("Failed reading manifest %s, %v\n", manifestPath, err) } + manifestFileBar.Finish() + var objects []ManifestObject err = json.Unmarshal(manifestBytes, &objects) if err != nil { log.Fatalf("Error has occurred during unmarshalling manifest object: %v\n", err) } - downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) + err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) + if err != nil { + log.Fatalln(err.Error()) + } err = logs.CloseMessageLog() if err != nil { - log.Println(err.Error()) + log.Fatalln(err.Error()) } }, } diff --git a/gen3-client/g3cmd/download-single.go b/data-client/g3cmd/download-single.go similarity index 83% rename from gen3-client/g3cmd/download-single.go rename to data-client/g3cmd/download-single.go index e4ea972..0a1b1ba 100644 --- a/gen3-client/g3cmd/download-single.go +++ b/data-client/g3cmd/download-single.go @@ -3,8 +3,8 @@ package g3cmd import ( "log" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func init() { @@ -20,18 +20,25 @@ func init() { Use: "download-single", Short: "Download a single file from a GUID", Long: `Gets a presigned URL for a file from a GUID and then downloads the specified file.`, - Example: `./gen3-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, + Example: `./data-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - profileConfig = conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + } obj := ManifestObject{ ObjectID: guid, } objects := []ManifestObject{obj} - downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) - err := logs.CloseMessageLog() + err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) + if err != nil { + log.Println(err.Error()) + } + err = logs.CloseMessageLog() if err != nil { log.Println(err.Error()) } diff --git a/gen3-client/g3cmd/generate-tsv.go b/data-client/g3cmd/generate-tsv.go similarity index 86% rename from gen3-client/g3cmd/generate-tsv.go rename to data-client/g3cmd/generate-tsv.go index af47a6d..9abff77 100644 --- a/gen3-client/g3cmd/generate-tsv.go +++ b/data-client/g3cmd/generate-tsv.go @@ -9,7 +9,7 @@ func init() { Use: "generate-tsv", Short: "Generate a file upload tsv from a template", Long: `Fills in a Gen3 data file template with information from a directory of files.`, - Deprecated: "please use an older version of gen3-client", + Deprecated: "please use an older version of data-client", Run: func(cmd *cobra.Command, args []string) {}, } diff --git a/gen3-client/g3cmd/gitversion.go b/data-client/g3cmd/gitversion.go similarity index 100% rename from gen3-client/g3cmd/gitversion.go rename to data-client/g3cmd/gitversion.go diff --git a/gen3-client/g3cmd/retry-upload.go b/data-client/g3cmd/retry-upload.go similarity index 93% rename from gen3-client/g3cmd/retry-upload.go rename to data-client/g3cmd/retry-upload.go index ad4bab3..2b4f404 100644 --- a/gen3-client/g3cmd/retry-upload.go +++ b/data-client/g3cmd/retry-upload.go @@ -8,8 +8,8 @@ import ( "time" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" ) func updateRetryObject(ro *commonUtils.RetryObject, filePath string, filename string, fileMetadata commonUtils.FileMetadata, guid string, retryCount int, isMultipart bool) { @@ -170,16 +170,24 @@ func init() { Use: "retry-upload", Short: "Retry upload file(s) to object storage.", Long: `Re-submit files found in a given failed log by using sequential (non-batching) uploading and exponential backoff.`, - Example: "For retrying file upload:\n./gen3-client retry-upload --profile= --failed-log-path=\n", + Example: "For retrying file upload:\n./data-client retry-upload --profile= --failed-log-path=\n", Run: func(cmd *cobra.Command, args []string) { // initialize transmission logs logs.InitSucceededLog(profile) logs.InitFailedLog(profile) logs.SetToBoth() logs.InitScoreBoard(MaxRetryCount) - profileConfig = conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + } - failedLogPath = commonUtils.ParseRootPath(failedLogPath) + failedLogPath, err = commonUtils.ParseRootPath(failedLogPath) + if err != nil { + log.Println(err.Error()) + return + } logs.LoadFailedLogFile(failedLogPath) retryUpload(logs.GetFailedLogMap()) logs.PrintScoreBoard() diff --git a/gen3-client/g3cmd/root.go b/data-client/g3cmd/root.go similarity index 86% rename from gen3-client/g3cmd/root.go rename to data-client/g3cmd/root.go index 8966390..d7e58c2 100644 --- a/gen3-client/g3cmd/root.go +++ b/data-client/g3cmd/root.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" latest "github.com/tcnksm/go-latest" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/logs" ) var profile string @@ -18,9 +18,9 @@ var profileConfig jwt.Credential // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ - Use: "gen3-client", - Short: "Use the gen3-client to interact with a Gen3 Data Commons", - Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ngen3-client version: " + gitversion + ", commit: " + gitcommit, + Use: "data-client", + Short: "Use the data-client to interact with a Gen3 Data Commons", + Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ndata-client version: " + gitversion + ", commit: " + gitcommit, Version: gitversion, } @@ -82,8 +82,8 @@ func initConfig() { if err != nil { log.Println("Error occurred when checking for latest version: " + err.Error()) } else if res.Outdated { - log.Println("A new version of gen3-client is available! The latest version is " + res.Current + ". You are using version " + gitversion) - log.Println("Please download the latest gen3-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") + log.Println("A new version of data-client is available! The latest version is " + res.Current + ". You are using version " + gitversion) + log.Println("Please download the latest data-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") } } logs.SetToMessageLog() diff --git a/gen3-client/g3cmd/upload-multipart.go b/data-client/g3cmd/upload-multipart.go similarity index 99% rename from gen3-client/g3cmd/upload-multipart.go rename to data-client/g3cmd/upload-multipart.go index f1dc1e6..66931ac 100644 --- a/gen3-client/g3cmd/upload-multipart.go +++ b/data-client/g3cmd/upload-multipart.go @@ -13,7 +13,7 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/logs" pb "gopkg.in/cheggaaa/pb.v1" ) diff --git a/gen3-client/g3cmd/upload-multiple.go b/data-client/g3cmd/upload-multiple.go similarity index 89% rename from gen3-client/g3cmd/upload-multiple.go rename to data-client/g3cmd/upload-multiple.go index 549f055..62584bb 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/data-client/g3cmd/upload-multiple.go @@ -4,15 +4,14 @@ package g3cmd import ( "encoding/json" "fmt" - "io/ioutil" "log" "os" "path/filepath" "strings" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func init() { @@ -28,13 +27,17 @@ func init() { Use: "upload-multiple", Short: "Upload multiple of files from a specified manifest", Long: `Get presigned URLs for multiple of files specified in a manifest file and then upload all of them. Options to run multipart uploads for large files and running multiple workers to batch upload available.`, - Example: `./gen3-client upload-multiple --profile= --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, + Example: `./data-client upload-multiple --profile= --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./gen3-client upload\" instead.\n\n") + fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./data-client upload\" instead.\n\n") // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Fatalln("Error occurred during parsing config file for hostname: " + err.Error()) + } host, err := gen3Interface.GetHost(&profileConfig) if err != nil { @@ -60,7 +63,7 @@ func init() { switch { case strings.EqualFold(filepath.Ext(manifestPath), ".json"): - manifestBytes, err := ioutil.ReadFile(manifestPath) + manifestBytes, err := os.ReadFile(manifestPath) if err != nil { log.Printf("Failed reading manifest %s, %v\n", manifestPath, err) log.Fatalln("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) @@ -76,7 +79,7 @@ func init() { uploadPath, err := commonUtils.GetAbsolutePath(uploadPath) if err != nil { - log.Fatalf("Error when parsing file paths: " + err.Error()) + log.Fatalf("Error when parsing file paths: %s", err.Error()) } filePaths := make([]string, 0) @@ -85,7 +88,7 @@ func init() { var err error if object.Filename != "" { - // conform to fence naming convention + // conform to fence naming convention filePath, err = getFullFilePath(uploadPath, object.Filename) } else { // Otherwise, here we are assuming the local filename will be the same as GUID @@ -126,7 +129,10 @@ func init() { processSingleUploads(gen3Interface, singlePartFilePaths, bucketName, includeSubDirName, uploadPath) } if len(multipartFilePaths) > 0 { - processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + err := processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + if err != nil { + log.Fatalln(err.Error()) + } } if !logs.IsFailedLogMapEmpty() { retryUpload(logs.GetFailedLogMap()) @@ -205,11 +211,15 @@ func startSingleFileUpload(gen3Interface Gen3Interface, filePath string, file *o file.Close() } -func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []string, bucketName string, includeSubDirName bool, uploadPath string) { - profileConfig := conf.ParseConfig(profile) +func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []string, bucketName string, includeSubDirName bool, uploadPath string) error { + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + return err + } if profileConfig.UseShepherd == "true" || profileConfig.UseShepherd == "" && commonUtils.DefaultUseShepherd == true { - log.Fatalf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) + return fmt.Errorf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ data-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) } log.Println("Multipart uploading....") @@ -227,4 +237,5 @@ func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []st logs.IncrementScore(0) } } + return nil } diff --git a/gen3-client/g3cmd/upload-single.go b/data-client/g3cmd/upload-single.go similarity index 78% rename from gen3-client/g3cmd/upload-single.go rename to data-client/g3cmd/upload-single.go index ec465a1..c4297ca 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/data-client/g3cmd/upload-single.go @@ -8,8 +8,8 @@ import ( "os" "path/filepath" + "github.com/calypr/data-client/data-client/commonUtils" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" ) func init() { @@ -21,10 +21,13 @@ func init() { Use: "upload-single", Short: "Upload a single file to a GUID", Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, - Example: `./gen3-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, + Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide a GUID. In this method file will be uploaded to a specified GUID.\nIf your intention is to upload file without pre-existing GUID, consider to use \"./gen3-client upload\" instead.\n\n") - UploadSingle(profile, guid, filePath, bucketName) + fmt.Printf("Notice: this is the upload method which requires the user to provide a GUID. In this method file will be uploaded to a specified GUID.\nIf your intention is to upload file without pre-existing GUID, consider to use \"./data-client upload\" instead.\n\n") + err := UploadSingle(profile, guid, filePath, bucketName) + if err != nil { + log.Fatalln(err.Error()) + } }, } uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") @@ -57,7 +60,10 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + return err + } log.Println("INSIDE: profile config parsed") @@ -68,7 +74,8 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin return errors.New(errorStr) } if err != nil { - log.Fatalln("File path parsing error: " + err.Error()) + log.Println("File path parsing error: " + err.Error()) + return err } if len(filePaths) == 1 { filePath = filePaths[0] @@ -80,21 +87,23 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() - log.Fatalf("The file you specified \"%s\" does not exist locally.", filePath) + errStr := fmt.Errorf("[ERROR] The file you specified \"%s\" does not exist locally.", filePath) + log.Println(errStr.Error()) + return errStr } log.Println("INSIDE: file path parsed") file, err := os.Open(filePath) if err != nil { - errorStr := fmt.Sprintf("[ERROR] when opening file path %s, an error occurred: %s", filePath, err.Error()) - log.Println(errorStr) + errorStr := fmt.Errorf("[ERROR] when opening file path %s, an error occurred: %s", filePath, err.Error()) + log.Println(errorStr.Error()) // logs.AddToFailedLog(filePath, filename, commonUtils.FileMetadata{}, "", 0, false, true) // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() // log.Fatalln("File open error: " + err.Error()) - return errors.New(errorStr) + return errorStr } defer file.Close() @@ -105,24 +114,24 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin furObject, err = GenerateUploadRequest(gen3Interface, furObject, file) if err != nil { file.Close() - errorStr := fmt.Sprintf("[ERROR] Error occurred during request generation for file %s: %s", filePath, err.Error()) - log.Println(errorStr) + errorStr := fmt.Errorf("[ERROR] Error occurred during request generation for file %s: %s", filePath, err.Error()) + log.Println(errorStr.Error()) // logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true) // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() // log.Fatalf("Error occurred during request generation: %s", err.Error()) - return errors.New(errorStr) + return errorStr } err = uploadFile(furObject, 0) if err != nil { - errStr := fmt.Sprintf("[ERROR] Error uploading file %s: %s", filePath, err.Error()) - log.Println(errStr) - return errors.New(errStr) + errStr := fmt.Errorf("[ERROR] Error uploading file %s: %s", filePath, err.Error()) + log.Println(errStr.Error()) + return errStr // logs.IncrementScore(logs.ScoreBoardLen - 1) // update failed score - } else { - // logs.IncrementScore(0) // update succeeded score - } + } /*else { + // logs.IncrementScore(0) // update succeeded score + }*/ // logs.PrintScoreBoard() // logs.CloseAll() log.Println("INSIDE: upload complete") diff --git a/gen3-client/g3cmd/upload.go b/data-client/g3cmd/upload.go similarity index 89% rename from gen3-client/g3cmd/upload.go rename to data-client/g3cmd/upload.go index 3e29083..6c22424 100644 --- a/gen3-client/g3cmd/upload.go +++ b/data-client/g3cmd/upload.go @@ -6,9 +6,9 @@ import ( "os" "path/filepath" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func init() { @@ -23,12 +23,12 @@ func init() { Use: "upload", Short: "Upload file(s) to object storage.", Long: `Gets a presigned URL for each file and then uploads the specified file(s).`, - Example: "For uploading a single file:\n./gen3-client upload --profile= --upload-path=\n" + - "For uploading all files within an folder:\n./gen3-client upload --profile= --upload-path=\n" + - "Can also support regex such as:\n./gen3-client upload --profile= --upload-path=\n" + - "Or:\n./gen3-client upload --profile= --upload-path=\n" + - "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the gen3-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + - "For example, if uploading the file `folder/my_file.bam`, the gen3-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + + Example: "For uploading a single file:\n./data-client upload --profile= --upload-path=\n" + + "For uploading all files within an folder:\n./data-client upload --profile= --upload-path=\n" + + "Can also support regex such as:\n./data-client upload --profile= --upload-path=\n" + + "Or:\n./data-client upload --profile= --upload-path=\n" + + "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the data-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + + "For example, if uploading the file `folder/my_file.bam`, the data-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + "For the format of the metadata files, see the README.", Run: func(cmd *cobra.Command, args []string) { // initialize transmission logs @@ -39,7 +39,12 @@ func init() { // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + return + } if hasMetadata { hasShepherd, err := gen3Interface.CheckForShepherdAPI(&profileConfig) @@ -55,7 +60,7 @@ func init() { uploadPath, _ = commonUtils.GetAbsolutePath(uploadPath) filePaths, err := commonUtils.ParseFilePaths(uploadPath, hasMetadata) if err != nil { - log.Fatalf("Error when parsing file paths: " + err.Error()) + log.Fatalf("Error when parsing file paths: %s", err.Error()) } if len(filePaths) == 0 { log.Println("No file has been found in the provided location \"" + uploadPath + "\"") @@ -154,7 +159,10 @@ func init() { if len(multipartFilePaths) > 0 { // NOTE(@mpingram) - For the moment Shepherd doesn't support multipart uploads. // Throw an error if Shepherd is enabled and user attempts to multipart upload. - processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + err := processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + if err != nil { + log.Println(err.Error()) + } } if !logs.IsFailedLogMapEmpty() { diff --git a/gen3-client/g3cmd/utils.go b/data-client/g3cmd/utils.go similarity index 98% rename from gen3-client/g3cmd/utils.go rename to data-client/g3cmd/utils.go index c2c68a3..3c3d8cd 100644 --- a/gen3-client/g3cmd/utils.go +++ b/data-client/g3cmd/utils.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "math" "net/http" @@ -18,14 +17,14 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" + "github.com/calypr/data-client/data-client/jwt" pb "gopkg.in/cheggaaa/pb.v1" ) -// go:generate mockgen -destination=./gen3-client/mocks/mock_gen3interface.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/g3cmd Gen3Interface +//go:generate mockgen -destination=./data-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/data-client/g3cmd Gen3Interface // ManifestObject represents an object from manifest that downloaded from windmill / data-portal type ManifestObject struct { @@ -497,7 +496,7 @@ func ProcessFilename(uploadPath string, filePath string, includeSubDirName bool, metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" var metadataFileBytes []byte if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = ioutil.ReadFile(metadataFilePath) + metadataFileBytes, err = os.ReadFile(metadataFilePath) if err != nil { return FileInfo{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) } @@ -507,7 +506,7 @@ func ProcessFilename(uploadPath string, filePath string, includeSubDirName bool, } } else { // No metadata file was found for this file -- proceed, but warn the user. - log.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `gen3-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) + log.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) } } return FileInfo{filePath, filename, metadata}, err diff --git a/gen3-client/jwt/configure.go b/data-client/jwt/configure.go similarity index 74% rename from gen3-client/jwt/configure.go rename to data-client/jwt/configure.go index 7f2532c..067e510 100644 --- a/gen3-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -1,12 +1,11 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_configure.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt ConfigureInterface +//go:generate mockgen -destination=./data-client/mocks/mock_configure.go -package=mocks github.com/calypr/data-client/data-client/jwt ConfigureInterface import ( "encoding/json" "errors" "fmt" - "io/ioutil" "log" "net/url" "os" @@ -14,11 +13,13 @@ import ( "regexp" "strings" + "github.com/calypr/data-client/data-client/commonUtils" homedir "github.com/mitchellh/go-homedir" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" "gopkg.in/ini.v1" ) +var ErrProfileNotFound = errors.New("profile not found in config file") + type Credential struct { Profile string KeyId string @@ -35,9 +36,9 @@ type ConfigureInterface interface { ReadFile(string, string) string ValidateUrl(string) (*url.URL, error) GetConfigPath() (string, error) - UpdateConfigFile(Credential) + UpdateConfigFile(Credential) error ParseKeyValue(str string, expr string) (string, error) - ParseConfig(profile string) Credential + ParseConfig(profile string) (Credential, error) } func (conf *Configure) ReadFile(filePath string, fileType string) string { @@ -52,7 +53,7 @@ func (conf *Configure) ReadFile(filePath string, fileType string) string { return "" } - content, err := ioutil.ReadFile(fullFilePath) + content, err := os.ReadFile(fullFilePath) if err != nil { log.Println("error occurred when reading file: " + err.Error()) return "" @@ -77,16 +78,18 @@ func (conf *Configure) ValidateUrl(apiEndpoint string) (*url.URL, error) { return parsedURL, nil } -func (conf *Configure) ReadCredentials(filePath string) Credential { +func (conf *Configure) ReadCredentials(filePath string) (*Credential, error) { var profileConfig Credential jsonContent := conf.ReadFile(filePath, "json") jsonContent = strings.Replace(jsonContent, "key_id", "KeyId", -1) jsonContent = strings.Replace(jsonContent, "api_key", "APIKey", -1) err := json.Unmarshal([]byte(jsonContent), &profileConfig) if err != nil { - log.Fatalln("Cannot read json file: " + err.Error()) + errs := fmt.Errorf("Cannot read json file: %s", err.Error()) + log.Println(errs.Error()) + return nil, errs } - return profileConfig + return &profileConfig, nil } func (conf *Configure) GetConfigPath() (string, error) { @@ -128,7 +131,7 @@ func (conf *Configure) InitConfigFile() error { return err } -func (conf *Configure) UpdateConfigFile(profileConfig Credential) { +func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { /* Overwrite the config file with new credential @@ -138,11 +141,15 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) { */ configPath, err := conf.GetConfigPath() if err != nil { - log.Fatalln("error occurred when getting config path: " + err.Error()) + errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) + log.Println(errs.Error()) + return errs } cfg, err := ini.Load(configPath) if err != nil { - log.Fatalln("error occurred when loading config file: " + err.Error()) + errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) + log.Println(errs.Error()) + return errs } cfg.Section(profileConfig.Profile).Key("key_id").SetValue(profileConfig.KeyId) cfg.Section(profileConfig.Profile).Key("api_key").SetValue(profileConfig.APIKey) @@ -152,8 +159,10 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) { cfg.Section(profileConfig.Profile).Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) err = cfg.SaveTo(configPath) if err != nil { - log.Println("error occurred when saving config file: " + err.Error()) + errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) + return errs } + return nil } func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { @@ -168,7 +177,7 @@ func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { return match[1], nil } -func (conf *Configure) ParseConfig(profile string) Credential { +func (conf *Configure) ParseConfig(profile string) (Credential, error) { /* Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can contain more than 1 profile. If there is no profile found, the user is asked to run a command to @@ -200,7 +209,8 @@ func (conf *Configure) ParseConfig(profile string) Credential { homeDir, err := homedir.Dir() if err != nil { - log.Fatalln("Error occurred when getting home directory: " + err.Error()) + errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) + return Credential{}, errs } configPath := path.Join(homeDir + commonUtils.PathSeparator + ".gen3" + commonUtils.PathSeparator + "gen3_client_config.ini") profileConfig := Credential{ @@ -211,41 +221,43 @@ func (conf *Configure) ParseConfig(profile string) Credential { APIEndpoint: "", } if _, err := os.Stat(configPath); os.IsNotExist(err) { - log.Println("No config file found in ~/.gen3/") - fmt.Println("Run configure command (with a profile if desired) to set up account credentials \n" + - "Example: ./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") - return profileConfig + return Credential{}, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ + "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) } // If profile not in config file, prompt user to set up config first cfg, err := ini.Load(configPath) if err != nil { - log.Fatalln("Error occurred when reading config file: " + err.Error()) + errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) + return Credential{}, errs } sec, err := cfg.GetSection(profile) - if err != nil { - log.Fatalln("Profile not in config file. Need to run \"gen3-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") + return Credential{}, fmt.Errorf("%w: Need to run \"data-client configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) } // Read in API key, key ID and endpoint for given profile profileConfig.KeyId = sec.Key("key_id").String() if profileConfig.KeyId == "" { - log.Fatalln("key_id not found in profile.") + errs := fmt.Errorf("key_id not found in profile.") + return Credential{}, errs } profileConfig.APIKey = sec.Key("api_key").String() if profileConfig.APIKey == "" { - log.Fatalln("api_key not found in profile.") + errs := fmt.Errorf("api_key not found in profile.") + return Credential{}, errs } profileConfig.AccessToken = sec.Key("access_token").String() if profileConfig.AccessToken == "" { - log.Fatalln("access_token not found in profile.") + errs := fmt.Errorf("access_token not found in profile.") + return Credential{}, errs } profileConfig.APIEndpoint = sec.Key("api_endpoint").String() if profileConfig.APIEndpoint == "" { - log.Fatalln("api_endpoint not found in profile.") + errs := fmt.Errorf("api_endpoint not found in profile.") + return Credential{}, errs } // UseShepherd and MinShepherdVersion are optional profileConfig.UseShepherd = sec.Key("use_shepherd").String() profileConfig.MinShepherdVersion = sec.Key("min_shepherd_version").String() - return profileConfig + return profileConfig, nil } diff --git a/gen3-client/jwt/functions.go b/data-client/jwt/functions.go similarity index 94% rename from gen3-client/jwt/functions.go rename to data-client/jwt/functions.go index eb6da0c..7184f22 100644 --- a/gen3-client/jwt/functions.go +++ b/data-client/jwt/functions.go @@ -1,22 +1,22 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_functions.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt FunctionInterface -//go:generate mockgen -destination=./gen3-client/mocks/mock_request.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt RequestInterface +//go:generate mockgen -destination=./data-client/mocks/mock_functions.go -package=mocks github.com/calypr/data-client/data-client/jwt FunctionInterface +//go:generate mockgen -destination=./data-client/mocks/mock_request.go -package=mocks github.com/calypr/data-client/data-client/jwt RequestInterface import ( "bytes" "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "log" "net/http" "net/url" "strconv" "strings" + "github.com/calypr/data-client/data-client/commonUtils" "github.com/hashicorp/go-version" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" ) type Functions struct { @@ -178,7 +178,7 @@ func (f *Functions) CheckForShepherdAPI(profileConfig *Credential) (bool, error) if res.StatusCode != 200 { return false, nil } - bodyBytes, err := ioutil.ReadAll(res.Body) + bodyBytes, err := io.ReadAll(res.Body) if err != nil { return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) } @@ -207,7 +207,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st var err error if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, errors.New("No credentials found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first") + return "", resp, fmt.Errorf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig) } host, _ := url.Parse(profileConfig.APIEndpoint) prefixEndPoint := host.Scheme + "://" + host.Host @@ -232,7 +232,10 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st if err != nil { return prefixEndPoint, resp, err } - f.Config.UpdateConfigFile(*profileConfig) + err = f.Config.UpdateConfigFile(*profileConfig) + if err != nil { + return prefixEndPoint, resp, err + } resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) if err != nil { @@ -245,7 +248,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { if profileConfig.APIEndpoint == "" { - return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first") + return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./data-client configure\" to configure your credentials first") } host, _ := url.Parse(profileConfig.APIEndpoint) return host, nil diff --git a/gen3-client/jwt/update.go b/data-client/jwt/update.go similarity index 77% rename from gen3-client/jwt/update.go rename to data-client/jwt/update.go index 75e775c..65e8826 100644 --- a/gen3-client/jwt/update.go +++ b/data-client/jwt/update.go @@ -5,9 +5,9 @@ import ( "log" "strings" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" "github.com/hashicorp/go-version" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func UpdateConfig(profile string, apiEndpoint string, credFile string, useShepherd string, minShepherdVersion string) error { @@ -15,7 +15,10 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe var conf Configure var req Request - profileConfig := conf.ReadCredentials(credFile) + profileConfig, err := conf.ReadCredentials(credFile) + if err != nil { + return err + } profileConfig.Profile = profile apiEndpoint = strings.TrimSpace(apiEndpoint) if apiEndpoint[len(apiEndpoint)-1:] == "/" { @@ -23,12 +26,11 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } parsedURL, err := conf.ValidateUrl(apiEndpoint) if err != nil { - // log.Fatalln("Error occurred when validating apiendpoint URL: " + err.Error()) - return fmt.Errorf("Errr occurred when validating apiendpoint URL: " + err.Error()) + return fmt.Errorf("Errr occurred when validating apiendpoint URL: %s", err.Error()) } prefixEndPoint := parsedURL.Scheme + "://" + parsedURL.Host - err = req.RequestNewAccessToken(prefixEndPoint+commonUtils.FenceAccessTokenEndpoint, &profileConfig) + err = req.RequestNewAccessToken(prefixEndPoint+commonUtils.FenceAccessTokenEndpoint, profileConfig) if err != nil { receivedErrorString := err.Error() errorMessageString := receivedErrorString @@ -37,8 +39,7 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } else if strings.Contains(receivedErrorString, "404") || strings.Contains(receivedErrorString, "405") || strings.Contains(receivedErrorString, "no such host") { errorMessageString = `The provided apiendpoint '` + prefixEndPoint + `' is possibly not a valid Gen3 data commons` } - // log.Fatalln("Error occurred when validating profile config: " + errorMessageString) - return fmt.Errorf("Error occurred when validating profile config: " + errorMessageString) + return fmt.Errorf("Error occurred when validating profile config: %s", errorMessageString) } profileConfig.APIEndpoint = apiEndpoint @@ -48,14 +49,16 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe if minShepherdVersion != "" { _, err = version.NewVersion(minShepherdVersion) if err != nil { - // log.Fatalln("Error occurred when validating minShepherdVersion: " + err.Error()) - return fmt.Errorf("Error occurred when validating minShepherdVersion: " + err.Error()) + return fmt.Errorf("Error occurred when validating minShepherdVersion: %s", err.Error()) } } profileConfig.MinShepherdVersion = minShepherdVersion // Store user info in ~/.gen3/gen3_client_config.ini - conf.UpdateConfigFile(profileConfig) + err = conf.UpdateConfigFile(*profileConfig) + if err != nil { + return err + } log.Println(`Profile '` + profile + `' has been configured successfully.`) err = logs.CloseMessageLog() if err != nil { diff --git a/gen3-client/jwt/utils.go b/data-client/jwt/utils.go similarity index 100% rename from gen3-client/jwt/utils.go rename to data-client/jwt/utils.go diff --git a/gen3-client/logs/failed.go b/data-client/logs/failed.go similarity index 97% rename from gen3-client/logs/failed.go rename to data-client/logs/failed.go index fbc25da..5f47fdf 100644 --- a/gen3-client/logs/failed.go +++ b/data-client/logs/failed.go @@ -2,13 +2,13 @@ package logs import ( "encoding/json" - "io/ioutil" + "io" "log" "os" "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" ) var failedLogFilename string @@ -47,7 +47,7 @@ func LoadFailedLogFile(filePath string) { if fi.Size() > 0 { var tempRetryObjectSlice []commonUtils.RetryObject - data, err := ioutil.ReadAll(file) + data, err := io.ReadAll(file) if err != nil { file.Close() failedLogFile.Close() diff --git a/gen3-client/logs/logs-master.go b/data-client/logs/logs-master.go similarity index 95% rename from gen3-client/logs/logs-master.go rename to data-client/logs/logs-master.go index 01b62e2..76dc8c2 100644 --- a/gen3-client/logs/logs-master.go +++ b/data-client/logs/logs-master.go @@ -4,8 +4,8 @@ import ( "log" "os" + "github.com/calypr/data-client/data-client/commonUtils" homedir "github.com/mitchellh/go-homedir" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" ) var MainLogPath string diff --git a/gen3-client/logs/scoreboard.go b/data-client/logs/scoreboard.go similarity index 99% rename from gen3-client/logs/scoreboard.go rename to data-client/logs/scoreboard.go index 85da3fa..492da1c 100644 --- a/gen3-client/logs/scoreboard.go +++ b/data-client/logs/scoreboard.go @@ -14,6 +14,7 @@ var ScoreBoardLen int func InitScoreBoard(maxRetryCount int) { scoreBoard = make([]int, maxRetryCount+2) ScoreBoardLen = len(scoreBoard) + } func PrintScoreBoard() { diff --git a/gen3-client/logs/succeeded.go b/data-client/logs/succeeded.go similarity index 97% rename from gen3-client/logs/succeeded.go rename to data-client/logs/succeeded.go index e010169..c1db33c 100644 --- a/gen3-client/logs/succeeded.go +++ b/data-client/logs/succeeded.go @@ -2,7 +2,7 @@ package logs import ( "encoding/json" - "io/ioutil" + "io" "log" "os" "sync" @@ -26,7 +26,7 @@ func InitSucceededLog(profile string) { succeededLogFileMap = make(map[string]string) if fi.Size() > 0 { - data, err := ioutil.ReadAll(succeededLogFile) + data, err := io.ReadAll(succeededLogFile) if err != nil { succeededLogFile.Close() log.Fatal("Error occurred when reading from file \"" + succeededLogFilename + "\": " + err.Error()) diff --git a/gen3-client/logs/system-msg.go b/data-client/logs/system-msg.go similarity index 100% rename from gen3-client/logs/system-msg.go rename to data-client/logs/system-msg.go diff --git a/gen3-client/mocks/mock_configure.go b/data-client/mocks/mock_configure.go similarity index 93% rename from gen3-client/mocks/mock_configure.go rename to data-client/mocks/mock_configure.go index a53cf9a..c379f68 100644 --- a/gen3-client/mocks/mock_configure.go +++ b/data-client/mocks/mock_configure.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: ConfigureInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: ConfigureInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" url "net/url" reflect "reflect" ) @@ -48,10 +48,10 @@ func (mr *MockConfigureInterfaceMockRecorder) GetConfigPath() *gomock.Call { } // ParseConfig mocks base method -func (m *MockConfigureInterface) ParseConfig(arg0 string) jwt.Credential { +func (m *MockConfigureInterface) ParseConfig(arg0 string) (jwt.Credential, error) { ret := m.ctrl.Call(m, "ParseConfig", arg0) ret0, _ := ret[0].(jwt.Credential) - return ret0 + return ret0, nil } // ParseConfig indicates an expected call of ParseConfig @@ -85,8 +85,9 @@ func (mr *MockConfigureInterfaceMockRecorder) ReadFile(arg0, arg1 interface{}) * } // UpdateConfigFile mocks base method -func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) { +func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) error { m.ctrl.Call(m, "UpdateConfigFile", arg0) + return nil } // UpdateConfigFile indicates an expected call of UpdateConfigFile diff --git a/gen3-client/mocks/mock_functions.go b/data-client/mocks/mock_functions.go similarity index 97% rename from gen3-client/mocks/mock_functions.go rename to data-client/mocks/mock_functions.go index 53a3245..51840e2 100644 --- a/gen3-client/mocks/mock_functions.go +++ b/data-client/mocks/mock_functions.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: FunctionInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: FunctionInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_gen3interface.go b/data-client/mocks/mock_gen3interface.go similarity index 97% rename from gen3-client/mocks/mock_gen3interface.go rename to data-client/mocks/mock_gen3interface.go index 3d39266..70f02d2 100644 --- a/gen3-client/mocks/mock_gen3interface.go +++ b/data-client/mocks/mock_gen3interface.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/g3cmd (interfaces: Gen3Interface) +// Source: github.com/calypr/data-client/data-client/g3cmd (interfaces: Gen3Interface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_request.go b/data-client/mocks/mock_request.go similarity index 94% rename from gen3-client/mocks/mock_request.go rename to data-client/mocks/mock_request.go index 9246bca..b627bf9 100644 --- a/gen3-client/mocks/mock_request.go +++ b/data-client/mocks/mock_request.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: RequestInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: RequestInterface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" reflect "reflect" ) diff --git a/go.mod b/go.mod index 210df5e..2d8ed1a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/uc-cdis/gen3-client +module github.com/calypr/data-client go 1.17 diff --git a/main.go b/main.go index 1383985..e482da8 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" + "github.com/calypr/data-client/data-client/g3cmd" ) func main() { diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index f1462e5..05fa734 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -2,16 +2,16 @@ package tests import ( "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" + "github.com/calypr/data-client/data-client/commonUtils" + g3cmd "github.com/calypr/data-client/data-client/g3cmd" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" ) // If Shepherd is deployed, attempt to get the filename from the Shepherd API. @@ -42,7 +42,7 @@ func Test_askGen3ForFileInfo_withShepherd(t *testing.T) { }` testResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(testBody)), + Body: io.NopCloser(strings.NewReader(testBody)), } mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) mockGen3Interface. diff --git a/tests/functions_test.go b/tests/functions_test.go index 884bcd5..3aa0609 100755 --- a/tests/functions_test.go +++ b/tests/functions_test.go @@ -3,15 +3,15 @@ package tests import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "reflect" "strings" "testing" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" ) func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { @@ -41,7 +41,7 @@ func TestDoRequestWithSignedHeaderGoodToken(t *testing.T) { profileConfig := jwt.Credential{Profile: "test", KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com", UseShepherd: "false", MinShepherdVersion: ""} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), StatusCode: 200, } @@ -65,7 +65,7 @@ func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), StatusCode: 200, } @@ -91,7 +91,7 @@ func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "expired_token", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), StatusCode: 401, } @@ -135,7 +135,7 @@ func TestCheckPrivilegesNoAccess(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), + Body: io.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), StatusCode: 200, } @@ -171,7 +171,7 @@ func TestCheckPrivilegesGrantedAccess(t *testing.T) { }` mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString(grantedAccessJSON)), + Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), StatusCode: 200, } @@ -225,7 +225,7 @@ func TestCheckPrivilegesGrantedAccessAuthz(t *testing.T) { }` mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString(grantedAccessJSON)), + Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), StatusCode: 200, } diff --git a/tests/utils_test.go b/tests/utils_test.go index 9aa834e..9875e02 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -3,16 +3,16 @@ package tests import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" + "github.com/calypr/data-client/data-client/commonUtils" + g3cmd "github.com/calypr/data-client/data-client/g3cmd" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" ) // Expect GetDownloadResponse to: @@ -43,7 +43,7 @@ func TestGetDownloadResponse_withShepherd(t *testing.T) { }`, mockDownloadURL) mockDownloadURLResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(downloadURLBody)), + Body: io.NopCloser(strings.NewReader(downloadURLBody)), } mockGen3Interface. EXPECT(). @@ -53,7 +53,7 @@ func TestGetDownloadResponse_withShepherd(t *testing.T) { // Mock the request for the file at mockDownloadURL. mockFileResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader("It work")), + Body: io.NopCloser(strings.NewReader("It work")), } mockGen3Interface. EXPECT(). @@ -112,7 +112,7 @@ func TestGetDownloadResponse_noShepherd(t *testing.T) { // Mock the request for the file at mockDownloadURL. mockFileResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader("It work")), + Body: io.NopCloser(strings.NewReader("It work")), } mockGen3Interface. EXPECT(). @@ -232,7 +232,7 @@ func TestGeneratePresignedURL_withShepherd(t *testing.T) { }`, mockGUID, mockPresignedURL) mockUploadURLResponse := http.Response{ StatusCode: 201, - Body: ioutil.NopCloser(strings.NewReader(presignedURLBody)), + Body: io.NopCloser(strings.NewReader(presignedURLBody)), } mockGen3Interface. EXPECT().