From 3852936baead7a30dae87485d4841b5dc074acb9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 21 Jul 2025 13:57:36 -0700 Subject: [PATCH 1/9] fix logging so that errors are passed to the drs logger --- gen3-client/commonUtils/commonUtils.go | 19 ++++---- gen3-client/g3cmd/auth.go | 7 ++- gen3-client/g3cmd/download-multiple.go | 31 +++++++------ gen3-client/g3cmd/download-single.go | 7 ++- gen3-client/g3cmd/retry-upload.go | 11 ++++- gen3-client/g3cmd/upload-multiple.go | 17 +++++--- gen3-client/g3cmd/upload-single.go | 30 +++++++------ gen3-client/g3cmd/upload.go | 8 +++- gen3-client/jwt/configure.go | 60 +++++++++++++++++--------- gen3-client/jwt/functions.go | 9 ++-- gen3-client/jwt/update.go | 10 ++--- 11 files changed, 134 insertions(+), 75 deletions(-) diff --git a/gen3-client/commonUtils/commonUtils.go b/gen3-client/commonUtils/commonUtils.go index 23a88f5..9dc49bc 100644 --- a/gen3-client/commonUtils/commonUtils.go +++ b/gen3-client/commonUtils/commonUtils.go @@ -74,7 +74,7 @@ type FileUploadRequestObject struct { PresignedURL string Request *http.Request Bar *pb.ProgressBar - Bucket string `json:"bucket,omitempty"` + Bucket string `json:"bucket,omitempty"` } // FileDownloadResponseObject defines a object for file download @@ -106,25 +106,28 @@ type RetryObject struct { GUID string RetryCount int Multipart bool - Bucket string + Bucket string } // ParseRootPath parses dirname that has "~" in the beginning -func ParseRootPath(filePath string) string { +func ParseRootPath(filePath string) (string, error) { if filePath != "" && filePath[0] == '~' { homeDir, err := homedir.Dir() if err != nil { - log.Fatalln(err) + return "", err } - return homeDir + filePath[1:] + return homeDir + filePath[1:], nil } - return filePath + return filePath, nil } // GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning func GetAbsolutePath(filePath string) (string, error) { - fullFilePath := ParseRootPath(filePath) - fullFilePath, err := filepath.Abs(fullFilePath) + fullFilePath, err := ParseRootPath(filePath) + if err != nil { + return fullFilePath, err + } + fullFilePath, err = filepath.Abs(fullFilePath) return fullFilePath, err } diff --git a/gen3-client/g3cmd/auth.go b/gen3-client/g3cmd/auth.go index cd8d4a2..39e4d43 100644 --- a/gen3-client/g3cmd/auth.go +++ b/gen3-client/g3cmd/auth.go @@ -20,9 +20,12 @@ func init() { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + profileConfig, err := conf.ParseConfig(profile) + if err != nil { + log.Fatalf("Fatal config parse error: %s\n", err) + } - host, resourceAccess, err := gen3Interface.CheckPrivileges(&profileConfig) + host, resourceAccess, err := gen3Interface.CheckPrivileges(profileConfig) if err != nil { log.Fatalf("Fatal authentication error: %s\n", err) diff --git a/gen3-client/g3cmd/download-multiple.go b/gen3-client/g3cmd/download-multiple.go index ed1c210..0b3507b 100644 --- a/gen3-client/g3cmd/download-multiple.go +++ b/gen3-client/g3cmd/download-multiple.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -158,9 +157,9 @@ func processOriginalFilename(downloadPath string, actualFilename string) string } } -func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) { +func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) error{ if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" { - log.Fatalln("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") + return fmt.Errorf("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") } if filenameFormat == "guid" || filenameFormat == "combined" { fmt.Printf("WARNING: in \"guid\" or \"combined\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) @@ -177,6 +176,7 @@ func validateFilenameFormat(downloadPath string, filenameFormat string, rename b } else { fmt.Printf("NOTICE: flag \"rename\" was set to true in \"original\" mode, duplicated files under \"%s\" will be renamed by appending a counter value to the original filenames\n", downloadPath) } + return nil } func validateLocalFileStat(downloadPath string, filename string, filesize int64, skipCompleted bool) commonUtils.FileDownloadResponseObject { @@ -284,12 +284,15 @@ func batchDownload(g3 Gen3Interface, batchFDRSlice []commonUtils.FileDownloadRes return succeeded } -func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) { +func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) error { if numParallel < 1 { - log.Fatalln("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.") + return fmt.Errorf("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.") } - downloadPath = commonUtils.ParseRootPath(downloadPath) + downloadPath, err := commonUtils.ParseRootPath(downloadPath) + if err != nil { + return fmt.Errorf("downloadFile Error: %s",err.Error()) + } if !strings.HasSuffix(downloadPath, "/") { downloadPath += "/" } @@ -305,9 +308,9 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat protocolText = "?protocol=" + protocol } - err := os.MkdirAll(downloadPath, 0766) - if err != nil { - log.Fatalln("Cannot create folder \"" + downloadPath + "\"") + err = os.MkdirAll(downloadPath, 0766) + if err != nil { + return fmt.Errorf("Cannot create folder \"" + downloadPath + "\"") } renamedFiles := make([]RenamedOrSkippedFileInfo, 0) @@ -381,6 +384,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat log.Println(err.Error()) } } + return nil } func init() { @@ -401,7 +405,6 @@ func init() { Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - profileConfig = conf.ParseConfig(profile) manifestPath, _ = commonUtils.GetAbsolutePath(manifestPath) manifestFile, err := os.Open(manifestPath) @@ -420,13 +423,13 @@ func init() { manifestFileReader := manifestFileBar.NewProxyReader(manifestFile) - manifestBytes, err := ioutil.ReadAll(manifestFileReader) - manifestFileBar.Finish() - + manifestBytes, err := io.ReadAll(manifestFileReader) if err != nil { log.Printf("Failed reading manifest %s, %v\n", manifestPath, err) - log.Fatalln("A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") + return } + manifestFileBar.Finish() + var objects []ManifestObject err = json.Unmarshal(manifestBytes, &objects) if err != nil { diff --git a/gen3-client/g3cmd/download-single.go b/gen3-client/g3cmd/download-single.go index e4ea972..a156059 100644 --- a/gen3-client/g3cmd/download-single.go +++ b/gen3-client/g3cmd/download-single.go @@ -24,14 +24,17 @@ func init() { Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - profileConfig = conf.ParseConfig(profile) + _, err := conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + } obj := ManifestObject{ ObjectID: guid, } objects := []ManifestObject{obj} downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) - err := logs.CloseMessageLog() + err = logs.CloseMessageLog() if err != nil { log.Println(err.Error()) } diff --git a/gen3-client/g3cmd/retry-upload.go b/gen3-client/g3cmd/retry-upload.go index ad4bab3..5f25c8f 100644 --- a/gen3-client/g3cmd/retry-upload.go +++ b/gen3-client/g3cmd/retry-upload.go @@ -177,9 +177,16 @@ func init() { logs.InitFailedLog(profile) logs.SetToBoth() logs.InitScoreBoard(MaxRetryCount) - profileConfig = conf.ParseConfig(profile) + _, err := conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + } - failedLogPath = commonUtils.ParseRootPath(failedLogPath) + failedLogPath, err = commonUtils.ParseRootPath(failedLogPath) + if err != nil { + log.Println(err.Error()) + return + } logs.LoadFailedLogFile(failedLogPath) retryUpload(logs.GetFailedLogMap()) logs.PrintScoreBoard() diff --git a/gen3-client/g3cmd/upload-multiple.go b/gen3-client/g3cmd/upload-multiple.go index 549f055..5afbc7a 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/gen3-client/g3cmd/upload-multiple.go @@ -34,9 +34,12 @@ func init() { // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + profileConfig, err := conf.ParseConfig(profile) + if err != nil { + log.Fatalln("Error occurred during parsing config file for hostname: " + err.Error()) + } - host, err := gen3Interface.GetHost(&profileConfig) + host, err := gen3Interface.GetHost(profileConfig) if err != nil { log.Fatalln("Error occurred during parsing config file for hostname: " + err.Error()) } @@ -205,11 +208,14 @@ func startSingleFileUpload(gen3Interface Gen3Interface, filePath string, file *o file.Close() } -func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []string, bucketName string, includeSubDirName bool, uploadPath string) { - profileConfig := conf.ParseConfig(profile) +func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []string, bucketName string, includeSubDirName bool, uploadPath string) error { + profileConfig, err := conf.ParseConfig(profile) + if err != nil { + return err + } if profileConfig.UseShepherd == "true" || profileConfig.UseShepherd == "" && commonUtils.DefaultUseShepherd == true { - log.Fatalf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) + return fmt.Errorf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) } log.Println("Multipart uploading....") @@ -227,4 +233,5 @@ func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []st logs.IncrementScore(0) } } + return nil } diff --git a/gen3-client/g3cmd/upload-single.go b/gen3-client/g3cmd/upload-single.go index ec465a1..2da2602 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/gen3-client/g3cmd/upload-single.go @@ -57,7 +57,10 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + _, err = conf.ParseConfig(profile) + if err != nil { + return err + } log.Println("INSIDE: profile config parsed") @@ -68,7 +71,8 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin return errors.New(errorStr) } if err != nil { - log.Fatalln("File path parsing error: " + err.Error()) + log.Println("File path parsing error: " + err.Error()) + return err } if len(filePaths) == 1 { filePath = filePaths[0] @@ -80,21 +84,23 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() - log.Fatalf("The file you specified \"%s\" does not exist locally.", filePath) + errStr := fmt.Errorf("[ERROR] The file you specified \"%s\" does not exist locally.", filePath) + log.Println(errStr.Error()) + return errStr } log.Println("INSIDE: file path parsed") file, err := os.Open(filePath) if err != nil { - errorStr := fmt.Sprintf("[ERROR] when opening file path %s, an error occurred: %s", filePath, err.Error()) - log.Println(errorStr) + errorStr := fmt.Errorf("[ERROR] when opening file path %s, an error occurred: %s", filePath, err.Error()) + log.Println(errorStr.Error()) // logs.AddToFailedLog(filePath, filename, commonUtils.FileMetadata{}, "", 0, false, true) // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() // log.Fatalln("File open error: " + err.Error()) - return errors.New(errorStr) + return errorStr } defer file.Close() @@ -105,20 +111,20 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin furObject, err = GenerateUploadRequest(gen3Interface, furObject, file) if err != nil { file.Close() - errorStr := fmt.Sprintf("[ERROR] Error occurred during request generation for file %s: %s", filePath, err.Error()) - log.Println(errorStr) + errorStr := fmt.Errorf("[ERROR] Error occurred during request generation for file %s: %s", filePath, err.Error()) + log.Println(errorStr.Error()) // logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true) // logs.IncrementScore(logs.ScoreBoardLen - 1) // logs.PrintScoreBoard() // logs.CloseAll() // log.Fatalf("Error occurred during request generation: %s", err.Error()) - return errors.New(errorStr) + return errorStr } err = uploadFile(furObject, 0) if err != nil { - errStr := fmt.Sprintf("[ERROR] Error uploading file %s: %s", filePath, err.Error()) - log.Println(errStr) - return errors.New(errStr) + errStr := fmt.Errorf("[ERROR] Error uploading file %s: %s", filePath, err.Error()) + log.Println(errStr.Error()) + return errStr // logs.IncrementScore(logs.ScoreBoardLen - 1) // update failed score } else { // logs.IncrementScore(0) // update succeeded score diff --git a/gen3-client/g3cmd/upload.go b/gen3-client/g3cmd/upload.go index 3e29083..e680c33 100644 --- a/gen3-client/g3cmd/upload.go +++ b/gen3-client/g3cmd/upload.go @@ -39,10 +39,14 @@ func init() { // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig = conf.ParseConfig(profile) + profileConfig, err := conf.ParseConfig(profile) + if err != nil { + log.Println(err.Error()) + return + } if hasMetadata { - hasShepherd, err := gen3Interface.CheckForShepherdAPI(&profileConfig) + hasShepherd, err := gen3Interface.CheckForShepherdAPI(profileConfig) if err != nil { log.Printf("WARNING: Error when checking for Shepherd API: %v", err) } else { diff --git a/gen3-client/jwt/configure.go b/gen3-client/jwt/configure.go index 7f2532c..ec49910 100644 --- a/gen3-client/jwt/configure.go +++ b/gen3-client/jwt/configure.go @@ -35,9 +35,9 @@ type ConfigureInterface interface { ReadFile(string, string) string ValidateUrl(string) (*url.URL, error) GetConfigPath() (string, error) - UpdateConfigFile(Credential) + UpdateConfigFile(*Credential) error ParseKeyValue(str string, expr string) (string, error) - ParseConfig(profile string) Credential + ParseConfig(profile string) (*Credential, error) } func (conf *Configure) ReadFile(filePath string, fileType string) string { @@ -77,16 +77,18 @@ func (conf *Configure) ValidateUrl(apiEndpoint string) (*url.URL, error) { return parsedURL, nil } -func (conf *Configure) ReadCredentials(filePath string) Credential { +func (conf *Configure) ReadCredentials(filePath string) (*Credential, error) { var profileConfig Credential jsonContent := conf.ReadFile(filePath, "json") jsonContent = strings.Replace(jsonContent, "key_id", "KeyId", -1) jsonContent = strings.Replace(jsonContent, "api_key", "APIKey", -1) err := json.Unmarshal([]byte(jsonContent), &profileConfig) if err != nil { - log.Fatalln("Cannot read json file: " + err.Error()) + errs := fmt.Errorf("Cannot read json file: " + err.Error()) + log.Println(errs.Error()) + return nil, errs } - return profileConfig + return &profileConfig, nil } func (conf *Configure) GetConfigPath() (string, error) { @@ -128,7 +130,7 @@ func (conf *Configure) InitConfigFile() error { return err } -func (conf *Configure) UpdateConfigFile(profileConfig Credential) { +func (conf *Configure) UpdateConfigFile(profileConfig *Credential) error{ /* Overwrite the config file with new credential @@ -138,11 +140,15 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) { */ configPath, err := conf.GetConfigPath() if err != nil { - log.Fatalln("error occurred when getting config path: " + err.Error()) + errs := fmt.Errorf("error occurred when getting config path: " + err.Error()) + log.Println(errs.Error()) + return errs } cfg, err := ini.Load(configPath) if err != nil { - log.Fatalln("error occurred when loading config file: " + err.Error()) + errs := fmt.Errorf("error occurred when loading config file: " + err.Error()) + log.Println(errs.Error()) + return errs } cfg.Section(profileConfig.Profile).Key("key_id").SetValue(profileConfig.KeyId) cfg.Section(profileConfig.Profile).Key("api_key").SetValue(profileConfig.APIKey) @@ -154,6 +160,7 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) { if err != nil { log.Println("error occurred when saving config file: " + err.Error()) } + return nil } func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { @@ -168,7 +175,7 @@ func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { return match[1], nil } -func (conf *Configure) ParseConfig(profile string) Credential { +func (conf *Configure) ParseConfig(profile string) (*Credential, error) { /* Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can contain more than 1 profile. If there is no profile found, the user is asked to run a command to @@ -200,10 +207,12 @@ func (conf *Configure) ParseConfig(profile string) Credential { homeDir, err := homedir.Dir() if err != nil { - log.Fatalln("Error occurred when getting home directory: " + err.Error()) + errs := fmt.Errorf("Error occurred when getting home directory: " + err.Error()) + log.Println(errs.Error()) + return nil, err } configPath := path.Join(homeDir + commonUtils.PathSeparator + ".gen3" + commonUtils.PathSeparator + "gen3_client_config.ini") - profileConfig := Credential{ + profileConfig := &Credential{ Profile: profile, KeyId: "", APIKey: "", @@ -214,38 +223,49 @@ func (conf *Configure) ParseConfig(profile string) Credential { log.Println("No config file found in ~/.gen3/") fmt.Println("Run configure command (with a profile if desired) to set up account credentials \n" + "Example: ./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") - return profileConfig + return nil, fmt.Errorf("No config file found in ~/.gen3/") } // If profile not in config file, prompt user to set up config first cfg, err := ini.Load(configPath) if err != nil { - log.Fatalln("Error occurred when reading config file: " + err.Error()) + errs := fmt.Errorf("Error occurred when reading config file: " + err.Error()) + log.Println(errs) + return nil, errs } sec, err := cfg.GetSection(profile) - if err != nil { - log.Fatalln("Profile not in config file. Need to run \"gen3-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") + errs := fmt.Errorf("Profile not in config file. Need to run \"gen3-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") + fmt.Println(errs.Error()) + return nil, err } // Read in API key, key ID and endpoint for given profile profileConfig.KeyId = sec.Key("key_id").String() if profileConfig.KeyId == "" { - log.Fatalln("key_id not found in profile.") + errs := fmt.Errorf("key_id not found in profile.") + log.Println(errs) + return nil, errs } profileConfig.APIKey = sec.Key("api_key").String() if profileConfig.APIKey == "" { - log.Fatalln("api_key not found in profile.") + errs := fmt.Errorf("api_key not found in profile.") + log.Println(errs) + return nil, errs } profileConfig.AccessToken = sec.Key("access_token").String() if profileConfig.AccessToken == "" { - log.Fatalln("access_token not found in profile.") + errs := fmt.Errorf("access_token not found in profile.") + log.Println(errs) + return nil, errs } profileConfig.APIEndpoint = sec.Key("api_endpoint").String() if profileConfig.APIEndpoint == "" { - log.Fatalln("api_endpoint not found in profile.") + errs := fmt.Errorf("api_endpoint not found in profile.") + log.Println(errs) + return nil, errs } // UseShepherd and MinShepherdVersion are optional profileConfig.UseShepherd = sec.Key("use_shepherd").String() profileConfig.MinShepherdVersion = sec.Key("min_shepherd_version").String() - return profileConfig + return profileConfig, nil } diff --git a/gen3-client/jwt/functions.go b/gen3-client/jwt/functions.go index eb6da0c..c736c18 100644 --- a/gen3-client/jwt/functions.go +++ b/gen3-client/jwt/functions.go @@ -8,8 +8,8 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "log" + "io" "net/http" "net/url" "strconv" @@ -178,7 +178,7 @@ func (f *Functions) CheckForShepherdAPI(profileConfig *Credential) (bool, error) if res.StatusCode != 200 { return false, nil } - bodyBytes, err := ioutil.ReadAll(res.Body) + bodyBytes, err := io.ReadAll(res.Body) if err != nil { return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) } @@ -232,7 +232,10 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st if err != nil { return prefixEndPoint, resp, err } - f.Config.UpdateConfigFile(*profileConfig) + err = f.Config.UpdateConfigFile(profileConfig) + if err != nil { + return prefixEndPoint, resp, err + } resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) if err != nil { diff --git a/gen3-client/jwt/update.go b/gen3-client/jwt/update.go index 75e775c..114ff47 100644 --- a/gen3-client/jwt/update.go +++ b/gen3-client/jwt/update.go @@ -15,7 +15,10 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe var conf Configure var req Request - profileConfig := conf.ReadCredentials(credFile) + profileConfig, err := conf.ReadCredentials(credFile) + if err != nil { + return err + } profileConfig.Profile = profile apiEndpoint = strings.TrimSpace(apiEndpoint) if apiEndpoint[len(apiEndpoint)-1:] == "/" { @@ -23,12 +26,11 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } parsedURL, err := conf.ValidateUrl(apiEndpoint) if err != nil { - // log.Fatalln("Error occurred when validating apiendpoint URL: " + err.Error()) return fmt.Errorf("Errr occurred when validating apiendpoint URL: " + err.Error()) } prefixEndPoint := parsedURL.Scheme + "://" + parsedURL.Host - err = req.RequestNewAccessToken(prefixEndPoint+commonUtils.FenceAccessTokenEndpoint, &profileConfig) + err = req.RequestNewAccessToken(prefixEndPoint+commonUtils.FenceAccessTokenEndpoint, profileConfig) if err != nil { receivedErrorString := err.Error() errorMessageString := receivedErrorString @@ -37,7 +39,6 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } else if strings.Contains(receivedErrorString, "404") || strings.Contains(receivedErrorString, "405") || strings.Contains(receivedErrorString, "no such host") { errorMessageString = `The provided apiendpoint '` + prefixEndPoint + `' is possibly not a valid Gen3 data commons` } - // log.Fatalln("Error occurred when validating profile config: " + errorMessageString) return fmt.Errorf("Error occurred when validating profile config: " + errorMessageString) } profileConfig.APIEndpoint = apiEndpoint @@ -48,7 +49,6 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe if minShepherdVersion != "" { _, err = version.NewVersion(minShepherdVersion) if err != nil { - // log.Fatalln("Error occurred when validating minShepherdVersion: " + err.Error()) return fmt.Errorf("Error occurred when validating minShepherdVersion: " + err.Error()) } } From a0186edcc84196cb996a8a150a4c058ae833af42 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 23 Jul 2025 08:53:51 -0700 Subject: [PATCH 2/9] fix parse config fun variable shadowing --- gen3-client/g3cmd/auth.go | 4 ++-- gen3-client/g3cmd/download-multiple.go | 13 +++++++++---- gen3-client/g3cmd/download-single.go | 3 ++- gen3-client/g3cmd/retry-upload.go | 3 ++- gen3-client/g3cmd/upload-multiple.go | 8 +++++--- gen3-client/g3cmd/upload-single.go | 2 +- gen3-client/g3cmd/upload.go | 5 +++-- gen3-client/g3cmd/utils.go | 14 +++++++------- gen3-client/jwt/configure.go | 26 +++++++++++++------------- gen3-client/jwt/functions.go | 10 +++++----- gen3-client/jwt/update.go | 2 +- 11 files changed, 50 insertions(+), 40 deletions(-) diff --git a/gen3-client/g3cmd/auth.go b/gen3-client/g3cmd/auth.go index 39e4d43..d70d88c 100644 --- a/gen3-client/g3cmd/auth.go +++ b/gen3-client/g3cmd/auth.go @@ -20,12 +20,12 @@ func init() { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() gen3Interface := NewGen3Interface() - profileConfig, err := conf.ParseConfig(profile) + profileConfig, err := conf.ParseConfig(profile) if err != nil { log.Fatalf("Fatal config parse error: %s\n", err) } - host, resourceAccess, err := gen3Interface.CheckPrivileges(profileConfig) + host, resourceAccess, err := gen3Interface.CheckPrivileges(&profileConfig) if err != nil { log.Fatalf("Fatal authentication error: %s\n", err) diff --git a/gen3-client/g3cmd/download-multiple.go b/gen3-client/g3cmd/download-multiple.go index 0b3507b..0d87bbf 100644 --- a/gen3-client/g3cmd/download-multiple.go +++ b/gen3-client/g3cmd/download-multiple.go @@ -68,7 +68,7 @@ func AskGen3ForFileInfo(gen3Interface Gen3Interface, guid string, protocol strin } else { // Attempt to get the filename from Indexd endPointPostfix := commonUtils.IndexdIndexEndpoint + "/" + guid - indexdMsg, err := gen3Interface.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "", nil) + indexdMsg, err := gen3Interface.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "", nil) if err != nil { log.Println("Error occurred when querying filename from IndexD: " + err.Error()) log.Println("Using GUID for filename instead.") @@ -157,7 +157,7 @@ func processOriginalFilename(downloadPath string, actualFilename string) string } } -func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) error{ +func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) error { if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" { return fmt.Errorf("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") } @@ -291,7 +291,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat downloadPath, err := commonUtils.ParseRootPath(downloadPath) if err != nil { - return fmt.Errorf("downloadFile Error: %s",err.Error()) + return fmt.Errorf("downloadFile Error: %s", err.Error()) } if !strings.HasSuffix(downloadPath, "/") { downloadPath += "/" @@ -309,7 +309,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat } err = os.MkdirAll(downloadPath, 0766) - if err != nil { + if err != nil { return fmt.Errorf("Cannot create folder \"" + downloadPath + "\"") } @@ -405,6 +405,11 @@ func init() { Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() + var err error + profileConfig, err = conf.ParseConfig(profile) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } manifestPath, _ = commonUtils.GetAbsolutePath(manifestPath) manifestFile, err := os.Open(manifestPath) diff --git a/gen3-client/g3cmd/download-single.go b/gen3-client/g3cmd/download-single.go index a156059..97a3472 100644 --- a/gen3-client/g3cmd/download-single.go +++ b/gen3-client/g3cmd/download-single.go @@ -24,7 +24,8 @@ func init() { Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - _, err := conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) if err != nil { log.Println(err.Error()) } diff --git a/gen3-client/g3cmd/retry-upload.go b/gen3-client/g3cmd/retry-upload.go index 5f25c8f..a536d57 100644 --- a/gen3-client/g3cmd/retry-upload.go +++ b/gen3-client/g3cmd/retry-upload.go @@ -177,7 +177,8 @@ func init() { logs.InitFailedLog(profile) logs.SetToBoth() logs.InitScoreBoard(MaxRetryCount) - _, err := conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) if err != nil { log.Println(err.Error()) } diff --git a/gen3-client/g3cmd/upload-multiple.go b/gen3-client/g3cmd/upload-multiple.go index 5afbc7a..9b041c3 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/gen3-client/g3cmd/upload-multiple.go @@ -34,12 +34,13 @@ func init() { // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig, err := conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) if err != nil { log.Fatalln("Error occurred during parsing config file for hostname: " + err.Error()) } - host, err := gen3Interface.GetHost(profileConfig) + host, err := gen3Interface.GetHost(&profileConfig) if err != nil { log.Fatalln("Error occurred during parsing config file for hostname: " + err.Error()) } @@ -209,7 +210,8 @@ func startSingleFileUpload(gen3Interface Gen3Interface, filePath string, file *o } func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []string, bucketName string, includeSubDirName bool, uploadPath string) error { - profileConfig, err := conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) if err != nil { return err } diff --git a/gen3-client/g3cmd/upload-single.go b/gen3-client/g3cmd/upload-single.go index 2da2602..4231e7a 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/gen3-client/g3cmd/upload-single.go @@ -57,7 +57,7 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - _, err = conf.ParseConfig(profile) + profileConfig, err = conf.ParseConfig(profile) if err != nil { return err } diff --git a/gen3-client/g3cmd/upload.go b/gen3-client/g3cmd/upload.go index e680c33..7ce80b1 100644 --- a/gen3-client/g3cmd/upload.go +++ b/gen3-client/g3cmd/upload.go @@ -39,14 +39,15 @@ func init() { // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() - profileConfig, err := conf.ParseConfig(profile) + var err error + profileConfig, err = conf.ParseConfig(profile) if err != nil { log.Println(err.Error()) return } if hasMetadata { - hasShepherd, err := gen3Interface.CheckForShepherdAPI(profileConfig) + hasShepherd, err := gen3Interface.CheckForShepherdAPI(&profileConfig) if err != nil { log.Printf("WARNING: Error when checking for Shepherd API: %v", err) } else { diff --git a/gen3-client/g3cmd/utils.go b/gen3-client/g3cmd/utils.go index c2c68a3..df59a05 100644 --- a/gen3-client/g3cmd/utils.go +++ b/gen3-client/g3cmd/utils.go @@ -131,7 +131,7 @@ func InitMultipartUpload(g3 Gen3Interface, filename string, bucketName string) ( return "", "", errors.New("Error has occurred during marshalling data for multipart upload initialization, detailed error message: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartInitEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartInitEndpoint, "application/json", objectBytes) if err != nil { if strings.Contains(err.Error(), "404") { @@ -160,7 +160,7 @@ func GenerateMultipartPresignedURL(g3 Gen3Interface, key string, uploadID string return "", errors.New("Error has occurred during marshalling data for multipart upload presigned url generation, detailed error message: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) if err != nil { return "", errors.New("Error has occurred during multipart upload presigned url generation, detailed error message: " + err.Error()) @@ -186,7 +186,7 @@ func CompleteMultipartUpload(g3 Gen3Interface, key string, uploadID string, part return errors.New("Error has occurred during marshalling data for multipart upload, detailed error message: " + err.Error()) } - _, err = g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) + _, err = g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) if err != nil { return errors.New("Error has occurred during completing multipart upload, detailed error message: " + err.Error()) } @@ -229,7 +229,7 @@ func GetDownloadResponse(g3 Gen3Interface, fdrObject *commonUtils.FileDownloadRe } } else { endPointPostfix := commonUtils.FenceDataDownloadEndpoint + "/" + fdrObject.GUID + protocolText - msg, err := g3.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "", nil) + msg, err := g3.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "", nil) if err != nil || msg.URL == "" { errorMsg := "Error occurred when getting download URL for object " + fdrObject.GUID @@ -335,7 +335,7 @@ func GeneratePresignedURL(g3 Gen3Interface, filename string, fileMetadata common if err != nil { return "", "", errors.New("Error occurred when marshalling object: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataUploadEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataUploadEndpoint, "application/json", objectBytes) if err != nil { return "", "", errors.New("Something went wrong. Maybe you don't have permission to upload data or Fence is misconfigured. Detailed error message: " + err.Error()) @@ -356,7 +356,7 @@ func GenerateUploadRequest(g3 Gen3Interface, furObject commonUtils.FileUploadReq endPointPostfix += "&bucket=" + furObject.Bucket } - msg, err := g3.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "application/json", nil) + msg, err := g3.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "application/json", nil) if err != nil && !strings.Contains(err.Error(), "No GUID found") { return furObject, errors.New("Upload error: " + err.Error()) } @@ -719,7 +719,7 @@ type Gen3Interface interface { CheckPrivileges(profileConfig *jwt.Credential) (string, map[string]interface{}, error) CheckForShepherdAPI(profileConfig *jwt.Credential) (bool, error) GetResponse(profileConfig *jwt.Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(profileConfig *jwt.Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) + DoRequestWithSignedHeader(profileConfig jwt.Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) GetHost(profileConfig *jwt.Credential) (*url.URL, error) } diff --git a/gen3-client/jwt/configure.go b/gen3-client/jwt/configure.go index ec49910..1a58ba3 100644 --- a/gen3-client/jwt/configure.go +++ b/gen3-client/jwt/configure.go @@ -35,9 +35,9 @@ type ConfigureInterface interface { ReadFile(string, string) string ValidateUrl(string) (*url.URL, error) GetConfigPath() (string, error) - UpdateConfigFile(*Credential) error + UpdateConfigFile(Credential) error ParseKeyValue(str string, expr string) (string, error) - ParseConfig(profile string) (*Credential, error) + ParseConfig(profile string) (Credential, error) } func (conf *Configure) ReadFile(filePath string, fileType string) string { @@ -130,7 +130,7 @@ func (conf *Configure) InitConfigFile() error { return err } -func (conf *Configure) UpdateConfigFile(profileConfig *Credential) error{ +func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { /* Overwrite the config file with new credential @@ -175,7 +175,7 @@ func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { return match[1], nil } -func (conf *Configure) ParseConfig(profile string) (*Credential, error) { +func (conf *Configure) ParseConfig(profile string) (Credential, error) { /* Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can contain more than 1 profile. If there is no profile found, the user is asked to run a command to @@ -209,10 +209,10 @@ func (conf *Configure) ParseConfig(profile string) (*Credential, error) { if err != nil { errs := fmt.Errorf("Error occurred when getting home directory: " + err.Error()) log.Println(errs.Error()) - return nil, err + return Credential{}, err } configPath := path.Join(homeDir + commonUtils.PathSeparator + ".gen3" + commonUtils.PathSeparator + "gen3_client_config.ini") - profileConfig := &Credential{ + profileConfig := Credential{ Profile: profile, KeyId: "", APIKey: "", @@ -223,7 +223,7 @@ func (conf *Configure) ParseConfig(profile string) (*Credential, error) { log.Println("No config file found in ~/.gen3/") fmt.Println("Run configure command (with a profile if desired) to set up account credentials \n" + "Example: ./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") - return nil, fmt.Errorf("No config file found in ~/.gen3/") + return Credential{}, fmt.Errorf("No config file found in ~/.gen3/") } // If profile not in config file, prompt user to set up config first @@ -231,38 +231,38 @@ func (conf *Configure) ParseConfig(profile string) (*Credential, error) { if err != nil { errs := fmt.Errorf("Error occurred when reading config file: " + err.Error()) log.Println(errs) - return nil, errs + return Credential{}, errs } sec, err := cfg.GetSection(profile) if err != nil { errs := fmt.Errorf("Profile not in config file. Need to run \"gen3-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") fmt.Println(errs.Error()) - return nil, err + return Credential{}, err } // Read in API key, key ID and endpoint for given profile profileConfig.KeyId = sec.Key("key_id").String() if profileConfig.KeyId == "" { errs := fmt.Errorf("key_id not found in profile.") log.Println(errs) - return nil, errs + return Credential{}, errs } profileConfig.APIKey = sec.Key("api_key").String() if profileConfig.APIKey == "" { errs := fmt.Errorf("api_key not found in profile.") log.Println(errs) - return nil, errs + return Credential{}, errs } profileConfig.AccessToken = sec.Key("access_token").String() if profileConfig.AccessToken == "" { errs := fmt.Errorf("access_token not found in profile.") log.Println(errs) - return nil, errs + return Credential{}, errs } profileConfig.APIEndpoint = sec.Key("api_endpoint").String() if profileConfig.APIEndpoint == "" { errs := fmt.Errorf("api_endpoint not found in profile.") log.Println(errs) - return nil, errs + return Credential{}, errs } // UseShepherd and MinShepherdVersion are optional profileConfig.UseShepherd = sec.Key("use_shepherd").String() diff --git a/gen3-client/jwt/functions.go b/gen3-client/jwt/functions.go index c736c18..b71b264 100644 --- a/gen3-client/jwt/functions.go +++ b/gen3-client/jwt/functions.go @@ -8,8 +8,8 @@ import ( "encoding/json" "errors" "fmt" - "log" "io" + "log" "net/http" "net/url" "strconv" @@ -207,7 +207,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st var err error if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, errors.New("No credentials found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first") + return "", resp, errors.New(fmt.Sprintf("No credentials found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first %s", profileConfig)) } host, _ := url.Parse(profileConfig.APIEndpoint) prefixEndPoint := host.Scheme + "://" + host.Host @@ -232,7 +232,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st if err != nil { return prefixEndPoint, resp, err } - err = f.Config.UpdateConfigFile(profileConfig) + err = f.Config.UpdateConfigFile(*profileConfig) if err != nil { return prefixEndPoint, resp, err } @@ -254,7 +254,7 @@ func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { return host, nil } -func (f *Functions) DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { +func (f *Functions) DoRequestWithSignedHeader(profileConfig Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { /* Do request with signed header. User may have more than one profile and use a profile to make a request */ @@ -266,7 +266,7 @@ func (f *Functions) DoRequestWithSignedHeader(profileConfig *Credential, endpoin method = "POST" } - _, resp, err := f.GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes) + _, resp, err := f.GetResponse(&profileConfig, endpointPostPrefix, method, contentType, bodyBytes) if err != nil { return msg, err } diff --git a/gen3-client/jwt/update.go b/gen3-client/jwt/update.go index 114ff47..022222d 100644 --- a/gen3-client/jwt/update.go +++ b/gen3-client/jwt/update.go @@ -55,7 +55,7 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe profileConfig.MinShepherdVersion = minShepherdVersion // Store user info in ~/.gen3/gen3_client_config.ini - conf.UpdateConfigFile(profileConfig) + conf.UpdateConfigFile(*profileConfig) log.Println(`Profile '` + profile + `' has been configured successfully.`) err = logs.CloseMessageLog() if err != nil { From 794674c758c3c357e0204af60d0e205564095506 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 28 Jul 2025 09:02:03 -0700 Subject: [PATCH 3/9] change names --- Dockerfile | 2 +- gen3-client/g3cmd/auth.go | 2 +- gen3-client/g3cmd/configure.go | 6 +++--- gen3-client/g3cmd/download-multiple.go | 4 ++-- gen3-client/g3cmd/download-single.go | 2 +- gen3-client/g3cmd/retry-upload.go | 4 ++-- gen3-client/g3cmd/root.go | 4 ++-- gen3-client/g3cmd/upload-multipart.go | 2 +- gen3-client/g3cmd/upload-multiple.go | 6 +++--- gen3-client/g3cmd/upload-single.go | 2 +- gen3-client/g3cmd/upload.go | 4 ++-- gen3-client/g3cmd/utils.go | 8 ++++---- gen3-client/jwt/configure.go | 4 ++-- gen3-client/jwt/functions.go | 6 +++--- gen3-client/jwt/update.go | 4 ++-- gen3-client/logs/failed.go | 2 +- gen3-client/logs/logs-master.go | 2 +- gen3-client/mocks/mock_configure.go | 4 ++-- gen3-client/mocks/mock_functions.go | 4 ++-- gen3-client/mocks/mock_gen3interface.go | 4 ++-- gen3-client/mocks/mock_request.go | 4 ++-- go.mod | 2 +- main.go | 2 +- tests/download-multiple_test.go | 8 ++++---- tests/functions_test.go | 4 ++-- tests/utils_test.go | 8 ++++---- 26 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8db04d5..84ddf45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV CGO_ENABLED=0 ENV GOOS=linux ENV GOARCH=amd64 -WORKDIR $GOPATH/src/github.com/uc-cdis/gen3-client/ +WORKDIR $GOPATH/src/github.com/calypr/gen3-client/ COPY go.mod . COPY go.sum . diff --git a/gen3-client/g3cmd/auth.go b/gen3-client/g3cmd/auth.go index d70d88c..b0d9328 100644 --- a/gen3-client/g3cmd/auth.go +++ b/gen3-client/g3cmd/auth.go @@ -6,7 +6,7 @@ import ( "sort" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/logs" ) func init() { diff --git a/gen3-client/g3cmd/configure.go b/gen3-client/g3cmd/configure.go index 5412c33..26b471b 100644 --- a/gen3-client/g3cmd/configure.go +++ b/gen3-client/g3cmd/configure.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/logs" ) var conf jwt.Configure // Why is this a global variable? diff --git a/gen3-client/g3cmd/download-multiple.go b/gen3-client/g3cmd/download-multiple.go index 0d87bbf..a41ad22 100644 --- a/gen3-client/g3cmd/download-multiple.go +++ b/gen3-client/g3cmd/download-multiple.go @@ -13,8 +13,8 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" pb "gopkg.in/cheggaaa/pb.v1" "github.com/spf13/cobra" diff --git a/gen3-client/g3cmd/download-single.go b/gen3-client/g3cmd/download-single.go index 97a3472..6bbba73 100644 --- a/gen3-client/g3cmd/download-single.go +++ b/gen3-client/g3cmd/download-single.go @@ -4,7 +4,7 @@ import ( "log" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/logs" ) func init() { diff --git a/gen3-client/g3cmd/retry-upload.go b/gen3-client/g3cmd/retry-upload.go index a536d57..fef9897 100644 --- a/gen3-client/g3cmd/retry-upload.go +++ b/gen3-client/g3cmd/retry-upload.go @@ -8,8 +8,8 @@ import ( "time" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" ) func updateRetryObject(ro *commonUtils.RetryObject, filePath string, filename string, fileMetadata commonUtils.FileMetadata, guid string, retryCount int, isMultipart bool) { diff --git a/gen3-client/g3cmd/root.go b/gen3-client/g3cmd/root.go index 8966390..6a3bae7 100644 --- a/gen3-client/g3cmd/root.go +++ b/gen3-client/g3cmd/root.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" latest "github.com/tcnksm/go-latest" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/logs" ) var profile string diff --git a/gen3-client/g3cmd/upload-multipart.go b/gen3-client/g3cmd/upload-multipart.go index f1dc1e6..7213be3 100644 --- a/gen3-client/g3cmd/upload-multipart.go +++ b/gen3-client/g3cmd/upload-multipart.go @@ -13,7 +13,7 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/logs" pb "gopkg.in/cheggaaa/pb.v1" ) diff --git a/gen3-client/g3cmd/upload-multiple.go b/gen3-client/g3cmd/upload-multiple.go index 9b041c3..ccf4ae6 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/gen3-client/g3cmd/upload-multiple.go @@ -10,9 +10,9 @@ import ( "path/filepath" "strings" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func init() { @@ -89,7 +89,7 @@ func init() { var err error if object.Filename != "" { - // conform to fence naming convention + // conform to fence naming convention filePath, err = getFullFilePath(uploadPath, object.Filename) } else { // Otherwise, here we are assuming the local filename will be the same as GUID diff --git a/gen3-client/g3cmd/upload-single.go b/gen3-client/g3cmd/upload-single.go index 4231e7a..677c472 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/gen3-client/g3cmd/upload-single.go @@ -9,7 +9,7 @@ import ( "path/filepath" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/commonUtils" ) func init() { diff --git a/gen3-client/g3cmd/upload.go b/gen3-client/g3cmd/upload.go index 7ce80b1..6d4d0eb 100644 --- a/gen3-client/g3cmd/upload.go +++ b/gen3-client/g3cmd/upload.go @@ -6,9 +6,9 @@ import ( "os" "path/filepath" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" "github.com/spf13/cobra" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" ) func init() { diff --git a/gen3-client/g3cmd/utils.go b/gen3-client/g3cmd/utils.go index df59a05..4616d59 100644 --- a/gen3-client/g3cmd/utils.go +++ b/gen3-client/g3cmd/utils.go @@ -18,14 +18,14 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/jwt" pb "gopkg.in/cheggaaa/pb.v1" ) -// go:generate mockgen -destination=./gen3-client/mocks/mock_gen3interface.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/g3cmd Gen3Interface +// go:generate mockgen -destination=./gen3-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/gen3-client/gen3-client/g3cmd Gen3Interface // ManifestObject represents an object from manifest that downloaded from windmill / data-portal type ManifestObject struct { diff --git a/gen3-client/jwt/configure.go b/gen3-client/jwt/configure.go index 1a58ba3..d8137db 100644 --- a/gen3-client/jwt/configure.go +++ b/gen3-client/jwt/configure.go @@ -1,6 +1,6 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_configure.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt ConfigureInterface +//go:generate mockgen -destination=./gen3-client/mocks/mock_configure.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt ConfigureInterface import ( "encoding/json" @@ -15,7 +15,7 @@ import ( "strings" homedir "github.com/mitchellh/go-homedir" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/commonUtils" "gopkg.in/ini.v1" ) diff --git a/gen3-client/jwt/functions.go b/gen3-client/jwt/functions.go index b71b264..1d8f5a9 100644 --- a/gen3-client/jwt/functions.go +++ b/gen3-client/jwt/functions.go @@ -1,7 +1,7 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_functions.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt FunctionInterface -//go:generate mockgen -destination=./gen3-client/mocks/mock_request.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/jwt RequestInterface +//go:generate mockgen -destination=./gen3-client/mocks/mock_functions.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt FunctionInterface +//go:generate mockgen -destination=./gen3-client/mocks/mock_request.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt RequestInterface import ( "bytes" @@ -15,8 +15,8 @@ import ( "strconv" "strings" + "github.com/calypr/gen3-client/gen3-client/commonUtils" "github.com/hashicorp/go-version" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" ) type Functions struct { diff --git a/gen3-client/jwt/update.go b/gen3-client/jwt/update.go index 022222d..54967f2 100644 --- a/gen3-client/jwt/update.go +++ b/gen3-client/jwt/update.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/hashicorp/go-version" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - "github.com/uc-cdis/gen3-client/gen3-client/logs" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/logs" ) func UpdateConfig(profile string, apiEndpoint string, credFile string, useShepherd string, minShepherdVersion string) error { diff --git a/gen3-client/logs/failed.go b/gen3-client/logs/failed.go index fbc25da..5106237 100644 --- a/gen3-client/logs/failed.go +++ b/gen3-client/logs/failed.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" + "github.com/calypr/gen3-client/gen3-client/commonUtils" ) var failedLogFilename string diff --git a/gen3-client/logs/logs-master.go b/gen3-client/logs/logs-master.go index 01b62e2..9bc00ef 100644 --- a/gen3-client/logs/logs-master.go +++ b/gen3-client/logs/logs-master.go @@ -4,8 +4,8 @@ import ( "log" "os" + "github.com/calypr/gen3-client/gen3-client/commonUtils" homedir "github.com/mitchellh/go-homedir" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" ) var MainLogPath string diff --git a/gen3-client/mocks/mock_configure.go b/gen3-client/mocks/mock_configure.go index a53cf9a..0429448 100644 --- a/gen3-client/mocks/mock_configure.go +++ b/gen3-client/mocks/mock_configure.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: ConfigureInterface) +// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: ConfigureInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/gen3-client/gen3-client/jwt" url "net/url" reflect "reflect" ) diff --git a/gen3-client/mocks/mock_functions.go b/gen3-client/mocks/mock_functions.go index 53a3245..ff41aab 100644 --- a/gen3-client/mocks/mock_functions.go +++ b/gen3-client/mocks/mock_functions.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: FunctionInterface) +// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: FunctionInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/gen3-client/gen3-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_gen3interface.go b/gen3-client/mocks/mock_gen3interface.go index 3d39266..5a79a52 100644 --- a/gen3-client/mocks/mock_gen3interface.go +++ b/gen3-client/mocks/mock_gen3interface.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/g3cmd (interfaces: Gen3Interface) +// Source: github.com/calypr/gen3-client/gen3-client/g3cmd (interfaces: Gen3Interface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/gen3-client/gen3-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_request.go b/gen3-client/mocks/mock_request.go index 9246bca..cf423ac 100644 --- a/gen3-client/mocks/mock_request.go +++ b/gen3-client/mocks/mock_request.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/uc-cdis/gen3-client/gen3-client/jwt (interfaces: RequestInterface) +// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: RequestInterface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/uc-cdis/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/gen3-client/gen3-client/jwt" http "net/http" reflect "reflect" ) diff --git a/go.mod b/go.mod index 210df5e..c523067 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/uc-cdis/gen3-client +module github.com/calypr/gen3-client go 1.17 diff --git a/main.go b/main.go index 1383985..44539a3 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" + "github.com/calypr/gen3-client/gen3-client/g3cmd" ) func main() { diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index f1462e5..386dfd9 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + g3cmd "github.com/calypr/gen3-client/gen3-client/g3cmd" + "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/mocks" ) // If Shepherd is deployed, attempt to get the filename from the Shepherd API. diff --git a/tests/functions_test.go b/tests/functions_test.go index 884bcd5..16a6723 100755 --- a/tests/functions_test.go +++ b/tests/functions_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" + "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/mocks" ) func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { diff --git a/tests/utils_test.go b/tests/utils_test.go index 9aa834e..18eb8db 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -9,10 +9,10 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/uc-cdis/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/uc-cdis/gen3-client/gen3-client/g3cmd" - "github.com/uc-cdis/gen3-client/gen3-client/jwt" - "github.com/uc-cdis/gen3-client/gen3-client/mocks" + "github.com/calypr/gen3-client/gen3-client/commonUtils" + g3cmd "github.com/calypr/gen3-client/gen3-client/g3cmd" + "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/gen3-client/gen3-client/mocks" ) // Expect GetDownloadResponse to: From a1424fbbb7885281ea4ac606278b3e31f6ce2b8d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 28 Jul 2025 09:08:52 -0700 Subject: [PATCH 4/9] rename --- Dockerfile | 10 +++--- README.md | 36 +++++++++---------- .../commonUtils/commonUtils.go | 4 +-- .../commonUtils/isHidden_notwindows.go | 0 .../commonUtils/isHidden_windows.go | 0 {gen3-client => data-client}/g3cmd/auth.go | 4 +-- .../g3cmd/configure.go | 8 ++--- {gen3-client => data-client}/g3cmd/delete.go | 4 +-- .../g3cmd/download-multiple.go | 6 ++-- .../g3cmd/download-single.go | 4 +-- .../g3cmd/generate-tsv.go | 2 +- .../g3cmd/gitversion.go | 0 .../g3cmd/retry-upload.go | 6 ++-- {gen3-client => data-client}/g3cmd/root.go | 14 ++++---- .../g3cmd/upload-multipart.go | 2 +- .../g3cmd/upload-multiple.go | 10 +++--- .../g3cmd/upload-single.go | 6 ++-- {gen3-client => data-client}/g3cmd/upload.go | 16 ++++----- {gen3-client => data-client}/g3cmd/utils.go | 10 +++--- {gen3-client => data-client}/jwt/configure.go | 8 ++--- {gen3-client => data-client}/jwt/functions.go | 10 +++--- {gen3-client => data-client}/jwt/update.go | 4 +-- {gen3-client => data-client}/jwt/utils.go | 0 {gen3-client => data-client}/logs/failed.go | 2 +- .../logs/logs-master.go | 2 +- .../logs/scoreboard.go | 0 .../logs/succeeded.go | 0 .../logs/system-msg.go | 0 .../mocks/mock_configure.go | 4 +-- .../mocks/mock_functions.go | 4 +-- .../mocks/mock_gen3interface.go | 4 +-- .../mocks/mock_request.go | 4 +-- go.mod | 2 +- main.go | 2 +- tests/download-multiple_test.go | 8 ++--- tests/functions_test.go | 4 +-- tests/utils_test.go | 8 ++--- 37 files changed, 104 insertions(+), 104 deletions(-) rename {gen3-client => data-client}/commonUtils/commonUtils.go (98%) rename {gen3-client => data-client}/commonUtils/isHidden_notwindows.go (100%) rename {gen3-client => data-client}/commonUtils/isHidden_windows.go (100%) rename {gen3-client => data-client}/g3cmd/auth.go (95%) rename {gen3-client => data-client}/g3cmd/configure.go (89%) rename {gen3-client => data-client}/g3cmd/delete.go (86%) rename {gen3-client => data-client}/g3cmd/download-multiple.go (99%) rename {gen3-client => data-client}/g3cmd/download-single.go (95%) rename {gen3-client => data-client}/g3cmd/generate-tsv.go (86%) rename {gen3-client => data-client}/g3cmd/gitversion.go (100%) rename {gen3-client => data-client}/g3cmd/retry-upload.go (97%) rename {gen3-client => data-client}/g3cmd/root.go (86%) rename {gen3-client => data-client}/g3cmd/upload-multipart.go (99%) rename {gen3-client => data-client}/g3cmd/upload-multiple.go (97%) rename {gen3-client => data-client}/g3cmd/upload-single.go (96%) rename {gen3-client => data-client}/g3cmd/upload.go (93%) rename {gen3-client => data-client}/g3cmd/utils.go (98%) rename {gen3-client => data-client}/jwt/configure.go (96%) rename {gen3-client => data-client}/jwt/functions.go (96%) rename {gen3-client => data-client}/jwt/update.go (95%) rename {gen3-client => data-client}/jwt/utils.go (100%) rename {gen3-client => data-client}/logs/failed.go (98%) rename {gen3-client => data-client}/logs/logs-master.go (95%) rename {gen3-client => data-client}/logs/scoreboard.go (100%) rename {gen3-client => data-client}/logs/succeeded.go (100%) rename {gen3-client => data-client}/logs/system-msg.go (100%) rename {gen3-client => data-client}/mocks/mock_configure.go (97%) rename {gen3-client => data-client}/mocks/mock_functions.go (97%) rename {gen3-client => data-client}/mocks/mock_gen3interface.go (97%) rename {gen3-client => data-client}/mocks/mock_request.go (95%) diff --git a/Dockerfile b/Dockerfile index 84ddf45..cc7000c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV CGO_ENABLED=0 ENV GOOS=linux ENV GOARCH=amd64 -WORKDIR $GOPATH/src/github.com/calypr/gen3-client/ +WORKDIR $GOPATH/src/github.com/calypr/data-client/ COPY go.mod . COPY go.sum . @@ -20,10 +20,10 @@ RUN COMMIT=$(git rev-parse HEAD); \ 'const ('\ ' gitcommit="'"${COMMIT}"'"'\ ' gitversion="'"${VERSION}"'"'\ - ')' > gen3-client/g3cmd/gitversion.go \ - && go build -o /gen3-client + ')' > data-client/g3cmd/gitversion.go \ + && go build -o /data-client FROM scratch COPY --from=build-deps /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=build-deps /gen3-client /gen3-client -CMD ["/gen3-client"] +COPY --from=build-deps /data-client /data-client +CMD ["/data-client"] diff --git a/README.md b/README.md index 48586e9..64ec2f3 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,24 @@ -# gen3-client +# data-client [![Build Status](https://travis-ci.org/uc-cdis/cdis-data-client.svg?branch=master)](https://travis-ci.org/uc-cdis/cdis-data-client) [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/uc-cdis/cdis-data-client?sort=semver)](https://github.com/uc-cdis/cdis-data-client/releases) -`gen3-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. +`data-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. -Read more about what it does and how to use it in the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/). +Read more about what it does and how to use it in the `data-client` [user guide](https://gen3.org/resources/user/data-client/). -`gen3-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra). +`data-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra). ## Installation -(The following instruction is for compiling and installing the `gen3-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases)) +(The following instruction is for compiling and installing the `data-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases)) First, [install Go and the Go tools](https://golang.org/doc/install) if you have not already done so. [Set up your workspace and your GOPATH.](https://golang.org/doc/code.html) Then: ``` -go get -d github.com/uc-cdis/gen3-client +go get -d github.com/calypr/data-client go install ``` @@ -30,43 +30,43 @@ go install mkdir -p $GOPATH/src/github.com/uc-cdis cd $GOPATH/src/github.com/uc-cdis git clone git@github.com:uc-cdis/cdis-data-client.git -mv cdis-data-client gen3-client -cd gen3-client +mv cdis-data-client data-client +cd data-client go get -d ./... go install . ``` -Now you should have `gen3-client` successfully installed. For a comprehensive instruction on how to configure and use `gen3-client` for uploading / downloading object files, please refer to the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/). +Now you should have `data-client` successfully installed. For a comprehensive instruction on how to configure and use `data-client` for uploading / downloading object files, please refer to the `data-client` [user guide](https://gen3.org/resources/user/data-client/). ## Enabling New Gen3 Object Management API Some Gen3 data commons support uploading files through the new Gen3 Object Management API. > NOTE: The service powering this API is sometimes referred to as our object "Shepherd" -To enable gen3-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `gen3-client configure`, e.g.: +To enable data-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `data-client configure`, e.g.: ``` -$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true +$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true ``` -If this flag is set, the gen3-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure. +If this flag is set, the data-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure. >You may also need to configure the version of the Gen3 Object Management API that the client will interact with. This is set to a default of Gen3 Object Management API `v2.0.0`, but can ->be raised or lowered by passing the `min-shepherd-version` flag to `gen3-client configure`, e.g.: +>be raised or lowered by passing the `min-shepherd-version` flag to `data-client configure`, e.g.: >``` ->$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0 +>$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0 >``` ### Uploading Additional File Object Metadata to Gen3 Object Management API -The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files. +The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files. > WARNING: Additional File Object Metadata is exposed publically and thus should not be controlled/sensitive data -You can upload file metadata using the `gen3-client upload` command with the `--metadata` flag. E.g.: +You can upload file metadata using the `data-client upload` command with the `--metadata` flag. E.g.: ``` -gen3-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata +data-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata ``` -This will tell `gen3-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`. +This will tell `data-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`. A metadata file should be located in the same folder as the file to be uploaded, and should be named `[filename]_metadata.json`. The metadata file should be a JSON file in the format: diff --git a/gen3-client/commonUtils/commonUtils.go b/data-client/commonUtils/commonUtils.go similarity index 98% rename from gen3-client/commonUtils/commonUtils.go rename to data-client/commonUtils/commonUtils.go index 9dc49bc..908825d 100644 --- a/gen3-client/commonUtils/commonUtils.go +++ b/data-client/commonUtils/commonUtils.go @@ -17,13 +17,13 @@ import ( // DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API // endpoints if available. -// The user can override this default using the `gen3-client configure` command. +// The user can override this default using the `data-client configure` command. const DefaultUseShepherd = false // DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use. // Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is // below this number the gen3client will instead warn the user and fall back to fence/indexd. -// The user can override this default using the `gen3-client configure` command. +// The user can override this default using the `data-client configure` command. const DefaultMinShepherdVersion = "2.0.0" // ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API diff --git a/gen3-client/commonUtils/isHidden_notwindows.go b/data-client/commonUtils/isHidden_notwindows.go similarity index 100% rename from gen3-client/commonUtils/isHidden_notwindows.go rename to data-client/commonUtils/isHidden_notwindows.go diff --git a/gen3-client/commonUtils/isHidden_windows.go b/data-client/commonUtils/isHidden_windows.go similarity index 100% rename from gen3-client/commonUtils/isHidden_windows.go rename to data-client/commonUtils/isHidden_windows.go diff --git a/gen3-client/g3cmd/auth.go b/data-client/g3cmd/auth.go similarity index 95% rename from gen3-client/g3cmd/auth.go rename to data-client/g3cmd/auth.go index b0d9328..af47e81 100644 --- a/gen3-client/g3cmd/auth.go +++ b/data-client/g3cmd/auth.go @@ -6,7 +6,7 @@ import ( "sort" "github.com/spf13/cobra" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/logs" ) func init() { @@ -15,7 +15,7 @@ func init() { Use: "auth", Short: "Return resource access privileges from profile", Long: `Gets resource access privileges for specified profile.`, - Example: `./gen3-client auth --profile=`, + Example: `./data-client auth --profile=`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() diff --git a/gen3-client/g3cmd/configure.go b/data-client/g3cmd/configure.go similarity index 89% rename from gen3-client/g3cmd/configure.go rename to data-client/g3cmd/configure.go index 26b471b..ea93a63 100644 --- a/gen3-client/g3cmd/configure.go +++ b/data-client/g3cmd/configure.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/spf13/cobra" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/jwt" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/logs" ) var conf jwt.Configure // Why is this a global variable? @@ -21,7 +21,7 @@ func init() { Short: "Add or modify a configuration profile to your config file", Long: `Configuration file located at ~/.gen3/gen3_client_config.ini If a field is left empty, the existing value (if it exists) will remain unchanged`, - Example: `./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, + Example: `./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() diff --git a/gen3-client/g3cmd/delete.go b/data-client/g3cmd/delete.go similarity index 86% rename from gen3-client/g3cmd/delete.go rename to data-client/g3cmd/delete.go index 95d90ee..5be6795 100644 --- a/gen3-client/g3cmd/delete.go +++ b/data-client/g3cmd/delete.go @@ -13,8 +13,8 @@ var deleteCmd = &cobra.Command{ // nolint:deadcode,unused,varcheck Short: "Send DELETE HTTP Request for given URI", Long: `Deletes a given URI from the database. If no profile is specified, "default" profile is used for authentication.`, - Example: `./gen3-client delete --uri=v0/submission/bpa/test/entities/example_id - ./gen3-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, + Example: `./data-client delete --uri=v0/submission/bpa/test/entities/example_id + ./data-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, Run: func(cmd *cobra.Command, args []string) { log.Fatalf("Not supported!") // request := new(jwt.Request) diff --git a/gen3-client/g3cmd/download-multiple.go b/data-client/g3cmd/download-multiple.go similarity index 99% rename from gen3-client/g3cmd/download-multiple.go rename to data-client/g3cmd/download-multiple.go index a41ad22..b1faeaa 100644 --- a/gen3-client/g3cmd/download-multiple.go +++ b/data-client/g3cmd/download-multiple.go @@ -13,8 +13,8 @@ import ( "sync" "time" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" pb "gopkg.in/cheggaaa/pb.v1" "github.com/spf13/cobra" @@ -401,7 +401,7 @@ func init() { Use: "download-multiple", Short: "Download multiple of files from a specified manifest", Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, - Example: `./gen3-client download-multiple --profile= --manifest= --download-path=`, + Example: `./data-client download-multiple --profile= --manifest= --download-path=`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() diff --git a/gen3-client/g3cmd/download-single.go b/data-client/g3cmd/download-single.go similarity index 95% rename from gen3-client/g3cmd/download-single.go rename to data-client/g3cmd/download-single.go index 6bbba73..af5ffc2 100644 --- a/gen3-client/g3cmd/download-single.go +++ b/data-client/g3cmd/download-single.go @@ -4,7 +4,7 @@ import ( "log" "github.com/spf13/cobra" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/logs" ) func init() { @@ -20,7 +20,7 @@ func init() { Use: "download-single", Short: "Download a single file from a GUID", Long: `Gets a presigned URL for a file from a GUID and then downloads the specified file.`, - Example: `./gen3-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, + Example: `./data-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() diff --git a/gen3-client/g3cmd/generate-tsv.go b/data-client/g3cmd/generate-tsv.go similarity index 86% rename from gen3-client/g3cmd/generate-tsv.go rename to data-client/g3cmd/generate-tsv.go index af47a6d..9abff77 100644 --- a/gen3-client/g3cmd/generate-tsv.go +++ b/data-client/g3cmd/generate-tsv.go @@ -9,7 +9,7 @@ func init() { Use: "generate-tsv", Short: "Generate a file upload tsv from a template", Long: `Fills in a Gen3 data file template with information from a directory of files.`, - Deprecated: "please use an older version of gen3-client", + Deprecated: "please use an older version of data-client", Run: func(cmd *cobra.Command, args []string) {}, } diff --git a/gen3-client/g3cmd/gitversion.go b/data-client/g3cmd/gitversion.go similarity index 100% rename from gen3-client/g3cmd/gitversion.go rename to data-client/g3cmd/gitversion.go diff --git a/gen3-client/g3cmd/retry-upload.go b/data-client/g3cmd/retry-upload.go similarity index 97% rename from gen3-client/g3cmd/retry-upload.go rename to data-client/g3cmd/retry-upload.go index fef9897..2b4f404 100644 --- a/gen3-client/g3cmd/retry-upload.go +++ b/data-client/g3cmd/retry-upload.go @@ -8,8 +8,8 @@ import ( "time" "github.com/spf13/cobra" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" ) func updateRetryObject(ro *commonUtils.RetryObject, filePath string, filename string, fileMetadata commonUtils.FileMetadata, guid string, retryCount int, isMultipart bool) { @@ -170,7 +170,7 @@ func init() { Use: "retry-upload", Short: "Retry upload file(s) to object storage.", Long: `Re-submit files found in a given failed log by using sequential (non-batching) uploading and exponential backoff.`, - Example: "For retrying file upload:\n./gen3-client retry-upload --profile= --failed-log-path=\n", + Example: "For retrying file upload:\n./data-client retry-upload --profile= --failed-log-path=\n", Run: func(cmd *cobra.Command, args []string) { // initialize transmission logs logs.InitSucceededLog(profile) diff --git a/gen3-client/g3cmd/root.go b/data-client/g3cmd/root.go similarity index 86% rename from gen3-client/g3cmd/root.go rename to data-client/g3cmd/root.go index 6a3bae7..d7e58c2 100644 --- a/gen3-client/g3cmd/root.go +++ b/data-client/g3cmd/root.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" latest "github.com/tcnksm/go-latest" - "github.com/calypr/gen3-client/gen3-client/jwt" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/logs" ) var profile string @@ -18,9 +18,9 @@ var profileConfig jwt.Credential // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ - Use: "gen3-client", - Short: "Use the gen3-client to interact with a Gen3 Data Commons", - Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ngen3-client version: " + gitversion + ", commit: " + gitcommit, + Use: "data-client", + Short: "Use the data-client to interact with a Gen3 Data Commons", + Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ndata-client version: " + gitversion + ", commit: " + gitcommit, Version: gitversion, } @@ -82,8 +82,8 @@ func initConfig() { if err != nil { log.Println("Error occurred when checking for latest version: " + err.Error()) } else if res.Outdated { - log.Println("A new version of gen3-client is available! The latest version is " + res.Current + ". You are using version " + gitversion) - log.Println("Please download the latest gen3-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") + log.Println("A new version of data-client is available! The latest version is " + res.Current + ". You are using version " + gitversion) + log.Println("Please download the latest data-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") } } logs.SetToMessageLog() diff --git a/gen3-client/g3cmd/upload-multipart.go b/data-client/g3cmd/upload-multipart.go similarity index 99% rename from gen3-client/g3cmd/upload-multipart.go rename to data-client/g3cmd/upload-multipart.go index 7213be3..66931ac 100644 --- a/gen3-client/g3cmd/upload-multipart.go +++ b/data-client/g3cmd/upload-multipart.go @@ -13,7 +13,7 @@ import ( "sync" "time" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/logs" pb "gopkg.in/cheggaaa/pb.v1" ) diff --git a/gen3-client/g3cmd/upload-multiple.go b/data-client/g3cmd/upload-multiple.go similarity index 97% rename from gen3-client/g3cmd/upload-multiple.go rename to data-client/g3cmd/upload-multiple.go index ccf4ae6..9834875 100644 --- a/gen3-client/g3cmd/upload-multiple.go +++ b/data-client/g3cmd/upload-multiple.go @@ -10,8 +10,8 @@ import ( "path/filepath" "strings" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" ) @@ -28,9 +28,9 @@ func init() { Use: "upload-multiple", Short: "Upload multiple of files from a specified manifest", Long: `Get presigned URLs for multiple of files specified in a manifest file and then upload all of them. Options to run multipart uploads for large files and running multiple workers to batch upload available.`, - Example: `./gen3-client upload-multiple --profile= --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, + Example: `./data-client upload-multiple --profile= --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./gen3-client upload\" instead.\n\n") + fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./data-client upload\" instead.\n\n") // Instantiate interface to Gen3 gen3Interface := NewGen3Interface() @@ -217,7 +217,7 @@ func processMultipartUpload(gen3Interface Gen3Interface, multipartFilePaths []st } if profileConfig.UseShepherd == "true" || profileConfig.UseShepherd == "" && commonUtils.DefaultUseShepherd == true { - return fmt.Errorf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) + return fmt.Errorf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ data-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile) } log.Println("Multipart uploading....") diff --git a/gen3-client/g3cmd/upload-single.go b/data-client/g3cmd/upload-single.go similarity index 96% rename from gen3-client/g3cmd/upload-single.go rename to data-client/g3cmd/upload-single.go index 677c472..63d7bfc 100644 --- a/gen3-client/g3cmd/upload-single.go +++ b/data-client/g3cmd/upload-single.go @@ -9,7 +9,7 @@ import ( "path/filepath" "github.com/spf13/cobra" - "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" ) func init() { @@ -21,9 +21,9 @@ func init() { Use: "upload-single", Short: "Upload a single file to a GUID", Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, - Example: `./gen3-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, + Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide a GUID. In this method file will be uploaded to a specified GUID.\nIf your intention is to upload file without pre-existing GUID, consider to use \"./gen3-client upload\" instead.\n\n") + fmt.Printf("Notice: this is the upload method which requires the user to provide a GUID. In this method file will be uploaded to a specified GUID.\nIf your intention is to upload file without pre-existing GUID, consider to use \"./data-client upload\" instead.\n\n") UploadSingle(profile, guid, filePath, bucketName) }, } diff --git a/gen3-client/g3cmd/upload.go b/data-client/g3cmd/upload.go similarity index 93% rename from gen3-client/g3cmd/upload.go rename to data-client/g3cmd/upload.go index 6d4d0eb..dde4723 100644 --- a/gen3-client/g3cmd/upload.go +++ b/data-client/g3cmd/upload.go @@ -6,8 +6,8 @@ import ( "os" "path/filepath" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" "github.com/spf13/cobra" ) @@ -23,12 +23,12 @@ func init() { Use: "upload", Short: "Upload file(s) to object storage.", Long: `Gets a presigned URL for each file and then uploads the specified file(s).`, - Example: "For uploading a single file:\n./gen3-client upload --profile= --upload-path=\n" + - "For uploading all files within an folder:\n./gen3-client upload --profile= --upload-path=\n" + - "Can also support regex such as:\n./gen3-client upload --profile= --upload-path=\n" + - "Or:\n./gen3-client upload --profile= --upload-path=\n" + - "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the gen3-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + - "For example, if uploading the file `folder/my_file.bam`, the gen3-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + + Example: "For uploading a single file:\n./data-client upload --profile= --upload-path=\n" + + "For uploading all files within an folder:\n./data-client upload --profile= --upload-path=\n" + + "Can also support regex such as:\n./data-client upload --profile= --upload-path=\n" + + "Or:\n./data-client upload --profile= --upload-path=\n" + + "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the data-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + + "For example, if uploading the file `folder/my_file.bam`, the data-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + "For the format of the metadata files, see the README.", Run: func(cmd *cobra.Command, args []string) { // initialize transmission logs diff --git a/gen3-client/g3cmd/utils.go b/data-client/g3cmd/utils.go similarity index 98% rename from gen3-client/g3cmd/utils.go rename to data-client/g3cmd/utils.go index 4616d59..9545441 100644 --- a/gen3-client/g3cmd/utils.go +++ b/data-client/g3cmd/utils.go @@ -18,14 +18,14 @@ import ( "sync" "time" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" - "github.com/calypr/gen3-client/gen3-client/jwt" + "github.com/calypr/data-client/data-client/jwt" pb "gopkg.in/cheggaaa/pb.v1" ) -// go:generate mockgen -destination=./gen3-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/gen3-client/gen3-client/g3cmd Gen3Interface +// go:generate mockgen -destination=./data-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/data-client/g3cmd Gen3Interface // ManifestObject represents an object from manifest that downloaded from windmill / data-portal type ManifestObject struct { @@ -507,7 +507,7 @@ func ProcessFilename(uploadPath string, filePath string, includeSubDirName bool, } } else { // No metadata file was found for this file -- proceed, but warn the user. - log.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `gen3-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) + log.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) } } return FileInfo{filePath, filename, metadata}, err diff --git a/gen3-client/jwt/configure.go b/data-client/jwt/configure.go similarity index 96% rename from gen3-client/jwt/configure.go rename to data-client/jwt/configure.go index d8137db..d625722 100644 --- a/gen3-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -1,6 +1,6 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_configure.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt ConfigureInterface +//go:generate mockgen -destination=./data-client/mocks/mock_configure.go -package=mocks github.com/calypr/data-client/data-client/jwt ConfigureInterface import ( "encoding/json" @@ -15,7 +15,7 @@ import ( "strings" homedir "github.com/mitchellh/go-homedir" - "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" "gopkg.in/ini.v1" ) @@ -222,7 +222,7 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { if _, err := os.Stat(configPath); os.IsNotExist(err) { log.Println("No config file found in ~/.gen3/") fmt.Println("Run configure command (with a profile if desired) to set up account credentials \n" + - "Example: ./gen3-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") + "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") return Credential{}, fmt.Errorf("No config file found in ~/.gen3/") } @@ -235,7 +235,7 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { } sec, err := cfg.GetSection(profile) if err != nil { - errs := fmt.Errorf("Profile not in config file. Need to run \"gen3-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") + errs := fmt.Errorf("Profile not in config file. Need to run \"data-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") fmt.Println(errs.Error()) return Credential{}, err } diff --git a/gen3-client/jwt/functions.go b/data-client/jwt/functions.go similarity index 96% rename from gen3-client/jwt/functions.go rename to data-client/jwt/functions.go index 1d8f5a9..54a7b70 100644 --- a/gen3-client/jwt/functions.go +++ b/data-client/jwt/functions.go @@ -1,7 +1,7 @@ package jwt -//go:generate mockgen -destination=./gen3-client/mocks/mock_functions.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt FunctionInterface -//go:generate mockgen -destination=./gen3-client/mocks/mock_request.go -package=mocks github.com/calypr/gen3-client/gen3-client/jwt RequestInterface +//go:generate mockgen -destination=./data-client/mocks/mock_functions.go -package=mocks github.com/calypr/data-client/data-client/jwt FunctionInterface +//go:generate mockgen -destination=./data-client/mocks/mock_request.go -package=mocks github.com/calypr/data-client/data-client/jwt RequestInterface import ( "bytes" @@ -15,7 +15,7 @@ import ( "strconv" "strings" - "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" "github.com/hashicorp/go-version" ) @@ -207,7 +207,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st var err error if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, errors.New(fmt.Sprintf("No credentials found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first %s", profileConfig)) + return "", resp, errors.New(fmt.Sprintf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig)) } host, _ := url.Parse(profileConfig.APIEndpoint) prefixEndPoint := host.Scheme + "://" + host.Host @@ -248,7 +248,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { if profileConfig.APIEndpoint == "" { - return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./gen3-client configure\" to configure your credentials first") + return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./data-client configure\" to configure your credentials first") } host, _ := url.Parse(profileConfig.APIEndpoint) return host, nil diff --git a/gen3-client/jwt/update.go b/data-client/jwt/update.go similarity index 95% rename from gen3-client/jwt/update.go rename to data-client/jwt/update.go index 54967f2..16650d4 100644 --- a/gen3-client/jwt/update.go +++ b/data-client/jwt/update.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/hashicorp/go-version" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - "github.com/calypr/gen3-client/gen3-client/logs" + "github.com/calypr/data-client/data-client/commonUtils" + "github.com/calypr/data-client/data-client/logs" ) func UpdateConfig(profile string, apiEndpoint string, credFile string, useShepherd string, minShepherdVersion string) error { diff --git a/gen3-client/jwt/utils.go b/data-client/jwt/utils.go similarity index 100% rename from gen3-client/jwt/utils.go rename to data-client/jwt/utils.go diff --git a/gen3-client/logs/failed.go b/data-client/logs/failed.go similarity index 98% rename from gen3-client/logs/failed.go rename to data-client/logs/failed.go index 5106237..94754ba 100644 --- a/gen3-client/logs/failed.go +++ b/data-client/logs/failed.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" ) var failedLogFilename string diff --git a/gen3-client/logs/logs-master.go b/data-client/logs/logs-master.go similarity index 95% rename from gen3-client/logs/logs-master.go rename to data-client/logs/logs-master.go index 9bc00ef..76dc8c2 100644 --- a/gen3-client/logs/logs-master.go +++ b/data-client/logs/logs-master.go @@ -4,7 +4,7 @@ import ( "log" "os" - "github.com/calypr/gen3-client/gen3-client/commonUtils" + "github.com/calypr/data-client/data-client/commonUtils" homedir "github.com/mitchellh/go-homedir" ) diff --git a/gen3-client/logs/scoreboard.go b/data-client/logs/scoreboard.go similarity index 100% rename from gen3-client/logs/scoreboard.go rename to data-client/logs/scoreboard.go diff --git a/gen3-client/logs/succeeded.go b/data-client/logs/succeeded.go similarity index 100% rename from gen3-client/logs/succeeded.go rename to data-client/logs/succeeded.go diff --git a/gen3-client/logs/system-msg.go b/data-client/logs/system-msg.go similarity index 100% rename from gen3-client/logs/system-msg.go rename to data-client/logs/system-msg.go diff --git a/gen3-client/mocks/mock_configure.go b/data-client/mocks/mock_configure.go similarity index 97% rename from gen3-client/mocks/mock_configure.go rename to data-client/mocks/mock_configure.go index 0429448..9f66a01 100644 --- a/gen3-client/mocks/mock_configure.go +++ b/data-client/mocks/mock_configure.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: ConfigureInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: ConfigureInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/calypr/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" url "net/url" reflect "reflect" ) diff --git a/gen3-client/mocks/mock_functions.go b/data-client/mocks/mock_functions.go similarity index 97% rename from gen3-client/mocks/mock_functions.go rename to data-client/mocks/mock_functions.go index ff41aab..51840e2 100644 --- a/gen3-client/mocks/mock_functions.go +++ b/data-client/mocks/mock_functions.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: FunctionInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: FunctionInterface) // Package mocks is a generated GoMock package. package mocks import ( gomock "github.com/golang/mock/gomock" - jwt "github.com/calypr/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_gen3interface.go b/data-client/mocks/mock_gen3interface.go similarity index 97% rename from gen3-client/mocks/mock_gen3interface.go rename to data-client/mocks/mock_gen3interface.go index 5a79a52..70f02d2 100644 --- a/gen3-client/mocks/mock_gen3interface.go +++ b/data-client/mocks/mock_gen3interface.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/gen3-client/gen3-client/g3cmd (interfaces: Gen3Interface) +// Source: github.com/calypr/data-client/data-client/g3cmd (interfaces: Gen3Interface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/calypr/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" url "net/url" reflect "reflect" diff --git a/gen3-client/mocks/mock_request.go b/data-client/mocks/mock_request.go similarity index 95% rename from gen3-client/mocks/mock_request.go rename to data-client/mocks/mock_request.go index cf423ac..b627bf9 100644 --- a/gen3-client/mocks/mock_request.go +++ b/data-client/mocks/mock_request.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/gen3-client/gen3-client/jwt (interfaces: RequestInterface) +// Source: github.com/calypr/data-client/data-client/jwt (interfaces: RequestInterface) // Package mocks is a generated GoMock package. package mocks @@ -7,7 +7,7 @@ package mocks import ( bytes "bytes" gomock "github.com/golang/mock/gomock" - jwt "github.com/calypr/gen3-client/gen3-client/jwt" + jwt "github.com/calypr/data-client/data-client/jwt" http "net/http" reflect "reflect" ) diff --git a/go.mod b/go.mod index c523067..2d8ed1a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/calypr/gen3-client +module github.com/calypr/data-client go 1.17 diff --git a/main.go b/main.go index 44539a3..e482da8 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/calypr/gen3-client/gen3-client/g3cmd" + "github.com/calypr/data-client/data-client/g3cmd" ) func main() { diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index 386dfd9..7e79003 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/calypr/gen3-client/gen3-client/g3cmd" - "github.com/calypr/gen3-client/gen3-client/jwt" - "github.com/calypr/gen3-client/gen3-client/mocks" + "github.com/calypr/data-client/data-client/commonUtils" + g3cmd "github.com/calypr/data-client/data-client/g3cmd" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" ) // If Shepherd is deployed, attempt to get the filename from the Shepherd API. diff --git a/tests/functions_test.go b/tests/functions_test.go index 16a6723..8dd3edd 100755 --- a/tests/functions_test.go +++ b/tests/functions_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/calypr/gen3-client/gen3-client/jwt" - "github.com/calypr/gen3-client/gen3-client/mocks" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" ) func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { diff --git a/tests/utils_test.go b/tests/utils_test.go index 18eb8db..8ee0da5 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -9,10 +9,10 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/calypr/gen3-client/gen3-client/commonUtils" - g3cmd "github.com/calypr/gen3-client/gen3-client/g3cmd" - "github.com/calypr/gen3-client/gen3-client/jwt" - "github.com/calypr/gen3-client/gen3-client/mocks" + "github.com/calypr/data-client/data-client/commonUtils" + g3cmd "github.com/calypr/data-client/data-client/g3cmd" + "github.com/calypr/data-client/data-client/jwt" + "github.com/calypr/data-client/data-client/mocks" ) // Expect GetDownloadResponse to: From 1c9d8e32b79a00eefa3fb5f39405e96f4baed275 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 30 Jul 2025 11:11:29 -0700 Subject: [PATCH 5/9] fix deprecated ioutil, fix tests --- data-client/.DS_Store | Bin 0 -> 6148 bytes data-client/g3cmd/configure.go | 8 ++++++-- data-client/g3cmd/upload-multiple.go | 3 +-- data-client/g3cmd/utils.go | 3 +-- data-client/jwt/configure.go | 5 ++--- data-client/logs/failed.go | 4 ++-- data-client/logs/scoreboard.go | 1 + data-client/logs/succeeded.go | 4 ++-- data-client/mocks/mock_configure.go | 7 ++++--- data-client/mocks/mock_gen3interface.go | 2 +- tests/download-multiple_test.go | 6 +++--- tests/functions_test.go | 24 ++++++++++++------------ tests/utils_test.go | 12 ++++++------ 13 files changed, 41 insertions(+), 38 deletions(-) create mode 100644 data-client/.DS_Store diff --git a/data-client/.DS_Store b/data-client/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4bc48646067ad752948dbf15c23b2cb12a4cd324 GIT binary patch literal 6148 zcmeHK%}T>S5Z<+|Z7D(y3Oz1(t=M3xh?h|73mDOZN=;0u!8BW%)Er77XMG``#OHBl zcLOa3k0N#kcE8#A+0A^A{b7vp!6L93vlwFyXowt@5)-$3&!R9%yWLN|Q>oT=>maTAgZnHqH}%p*GVtbiIJ%ZH@(bPb zZ^Cglvi2`zoO)qAo~wi~7(>YYZ5RhK8^}c*Br4a{jwZ;+YA=`le!nA*>{UlBZTr;j zh_haAwQ4pF4v){TMlaDcmT#I!4vZ_=Gg!boC~FnHy7M@a@dT_zMiEI!3=jjv05Pzg z44Ctv(b~@PscK??82Et!+#duqM8{yRQEeU2;q@8)RYVlf@hyQU3_1pLjo<;{Iu%f- za`VLCIvwo7#5o3YjXIriwKB|ORxTbdT&)gvp~4w=G*V9t5CfYGRP@lo^ZyclnbJr8 zW(tjn0b<~vF~F-scQ}Be%-Q;_JUnX!v`1(t7?-000{X@!00y{^w3So)1?mvz7|b=| TENEBhfOHX1giuEe`~m}Ce#S~d literal 0 HcmV?d00001 diff --git a/data-client/g3cmd/configure.go b/data-client/g3cmd/configure.go index ea93a63..0996121 100644 --- a/data-client/g3cmd/configure.go +++ b/data-client/g3cmd/configure.go @@ -2,11 +2,12 @@ package g3cmd import ( "fmt" + "log" - "github.com/spf13/cobra" "github.com/calypr/data-client/data-client/commonUtils" "github.com/calypr/data-client/data-client/jwt" "github.com/calypr/data-client/data-client/logs" + "github.com/spf13/cobra" ) var conf jwt.Configure // Why is this a global variable? @@ -26,7 +27,10 @@ func init() { // don't initialize transmission logs for non-uploading related commands logs.SetToBoth() - jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion) + err := jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion) + if err != nil { + log.Println(err.Error()) + } }, } diff --git a/data-client/g3cmd/upload-multiple.go b/data-client/g3cmd/upload-multiple.go index 9834875..b74f5f3 100644 --- a/data-client/g3cmd/upload-multiple.go +++ b/data-client/g3cmd/upload-multiple.go @@ -4,7 +4,6 @@ package g3cmd import ( "encoding/json" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -64,7 +63,7 @@ func init() { switch { case strings.EqualFold(filepath.Ext(manifestPath), ".json"): - manifestBytes, err := ioutil.ReadFile(manifestPath) + manifestBytes, err := os.ReadFile(manifestPath) if err != nil { log.Printf("Failed reading manifest %s, %v\n", manifestPath, err) log.Fatalln("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) diff --git a/data-client/g3cmd/utils.go b/data-client/g3cmd/utils.go index 9545441..77fa7d7 100644 --- a/data-client/g3cmd/utils.go +++ b/data-client/g3cmd/utils.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "math" "net/http" @@ -497,7 +496,7 @@ func ProcessFilename(uploadPath string, filePath string, includeSubDirName bool, metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" var metadataFileBytes []byte if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = ioutil.ReadFile(metadataFilePath) + metadataFileBytes, err = os.ReadFile(metadataFilePath) if err != nil { return FileInfo{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) } diff --git a/data-client/jwt/configure.go b/data-client/jwt/configure.go index d625722..202763d 100644 --- a/data-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "log" "net/url" "os" @@ -14,8 +13,8 @@ import ( "regexp" "strings" - homedir "github.com/mitchellh/go-homedir" "github.com/calypr/data-client/data-client/commonUtils" + homedir "github.com/mitchellh/go-homedir" "gopkg.in/ini.v1" ) @@ -52,7 +51,7 @@ func (conf *Configure) ReadFile(filePath string, fileType string) string { return "" } - content, err := ioutil.ReadFile(fullFilePath) + content, err := os.ReadFile(fullFilePath) if err != nil { log.Println("error occurred when reading file: " + err.Error()) return "" diff --git a/data-client/logs/failed.go b/data-client/logs/failed.go index 94754ba..5f47fdf 100644 --- a/data-client/logs/failed.go +++ b/data-client/logs/failed.go @@ -2,7 +2,7 @@ package logs import ( "encoding/json" - "io/ioutil" + "io" "log" "os" "sync" @@ -47,7 +47,7 @@ func LoadFailedLogFile(filePath string) { if fi.Size() > 0 { var tempRetryObjectSlice []commonUtils.RetryObject - data, err := ioutil.ReadAll(file) + data, err := io.ReadAll(file) if err != nil { file.Close() failedLogFile.Close() diff --git a/data-client/logs/scoreboard.go b/data-client/logs/scoreboard.go index 85da3fa..492da1c 100644 --- a/data-client/logs/scoreboard.go +++ b/data-client/logs/scoreboard.go @@ -14,6 +14,7 @@ var ScoreBoardLen int func InitScoreBoard(maxRetryCount int) { scoreBoard = make([]int, maxRetryCount+2) ScoreBoardLen = len(scoreBoard) + } func PrintScoreBoard() { diff --git a/data-client/logs/succeeded.go b/data-client/logs/succeeded.go index e010169..c1db33c 100644 --- a/data-client/logs/succeeded.go +++ b/data-client/logs/succeeded.go @@ -2,7 +2,7 @@ package logs import ( "encoding/json" - "io/ioutil" + "io" "log" "os" "sync" @@ -26,7 +26,7 @@ func InitSucceededLog(profile string) { succeededLogFileMap = make(map[string]string) if fi.Size() > 0 { - data, err := ioutil.ReadAll(succeededLogFile) + data, err := io.ReadAll(succeededLogFile) if err != nil { succeededLogFile.Close() log.Fatal("Error occurred when reading from file \"" + succeededLogFilename + "\": " + err.Error()) diff --git a/data-client/mocks/mock_configure.go b/data-client/mocks/mock_configure.go index 9f66a01..c379f68 100644 --- a/data-client/mocks/mock_configure.go +++ b/data-client/mocks/mock_configure.go @@ -48,10 +48,10 @@ func (mr *MockConfigureInterfaceMockRecorder) GetConfigPath() *gomock.Call { } // ParseConfig mocks base method -func (m *MockConfigureInterface) ParseConfig(arg0 string) jwt.Credential { +func (m *MockConfigureInterface) ParseConfig(arg0 string) (jwt.Credential, error) { ret := m.ctrl.Call(m, "ParseConfig", arg0) ret0, _ := ret[0].(jwt.Credential) - return ret0 + return ret0, nil } // ParseConfig indicates an expected call of ParseConfig @@ -85,8 +85,9 @@ func (mr *MockConfigureInterfaceMockRecorder) ReadFile(arg0, arg1 interface{}) * } // UpdateConfigFile mocks base method -func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) { +func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) error { m.ctrl.Call(m, "UpdateConfigFile", arg0) + return nil } // UpdateConfigFile indicates an expected call of UpdateConfigFile diff --git a/data-client/mocks/mock_gen3interface.go b/data-client/mocks/mock_gen3interface.go index 70f02d2..aed5c01 100644 --- a/data-client/mocks/mock_gen3interface.go +++ b/data-client/mocks/mock_gen3interface.go @@ -64,7 +64,7 @@ func (mr *MockGen3InterfaceMockRecorder) CheckPrivileges(arg0 interface{}) *gomo } // DoRequestWithSignedHeader mocks base method -func (m *MockGen3Interface) DoRequestWithSignedHeader(arg0 *jwt.Credential, arg1, arg2 string, arg3 []byte) (jwt.JsonMessage, error) { +func (m *MockGen3Interface) DoRequestWithSignedHeader(arg0 jwt.Credential, arg1, arg2 string, arg3 []byte) (jwt.JsonMessage, error) { ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(jwt.JsonMessage) ret1, _ := ret[1].(error) diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index 7e79003..05fa734 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -2,16 +2,16 @@ package tests import ( "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" - "github.com/golang/mock/gomock" "github.com/calypr/data-client/data-client/commonUtils" g3cmd "github.com/calypr/data-client/data-client/g3cmd" "github.com/calypr/data-client/data-client/jwt" "github.com/calypr/data-client/data-client/mocks" + "github.com/golang/mock/gomock" ) // If Shepherd is deployed, attempt to get the filename from the Shepherd API. @@ -42,7 +42,7 @@ func Test_askGen3ForFileInfo_withShepherd(t *testing.T) { }` testResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(testBody)), + Body: io.NopCloser(strings.NewReader(testBody)), } mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) mockGen3Interface. diff --git a/tests/functions_test.go b/tests/functions_test.go index 8dd3edd..70c64e6 100755 --- a/tests/functions_test.go +++ b/tests/functions_test.go @@ -3,15 +3,15 @@ package tests import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "reflect" "strings" "testing" - "github.com/golang/mock/gomock" "github.com/calypr/data-client/data-client/jwt" "github.com/calypr/data-client/data-client/mocks" + "github.com/golang/mock/gomock" ) func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { @@ -24,7 +24,7 @@ func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) if err == nil { t.Fail() @@ -41,13 +41,13 @@ func TestDoRequestWithSignedHeaderGoodToken(t *testing.T) { profileConfig := jwt.Credential{Profile: "test", KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com", UseShepherd: "false", MinShepherdVersion: ""} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), StatusCode: 200, } mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/data/download/test_uuid", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil { t.Fail() @@ -65,7 +65,7 @@ func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), StatusCode: 200, } @@ -73,7 +73,7 @@ func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil { t.Fail() @@ -91,7 +91,7 @@ func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "expired_token", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), + Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), StatusCode: 401, } @@ -99,7 +99,7 @@ func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(2) - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil && !strings.Contains(err.Error(), "401") { t.Fail() @@ -135,7 +135,7 @@ func TestCheckPrivilegesNoAccess(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), + Body: io.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), StatusCode: 200, } @@ -171,7 +171,7 @@ func TestCheckPrivilegesGrantedAccess(t *testing.T) { }` mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString(grantedAccessJSON)), + Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), StatusCode: 200, } @@ -225,7 +225,7 @@ func TestCheckPrivilegesGrantedAccessAuthz(t *testing.T) { }` mockedResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewBufferString(grantedAccessJSON)), + Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), StatusCode: 200, } diff --git a/tests/utils_test.go b/tests/utils_test.go index 8ee0da5..9875e02 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -3,16 +3,16 @@ package tests import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" - "github.com/golang/mock/gomock" "github.com/calypr/data-client/data-client/commonUtils" g3cmd "github.com/calypr/data-client/data-client/g3cmd" "github.com/calypr/data-client/data-client/jwt" "github.com/calypr/data-client/data-client/mocks" + "github.com/golang/mock/gomock" ) // Expect GetDownloadResponse to: @@ -43,7 +43,7 @@ func TestGetDownloadResponse_withShepherd(t *testing.T) { }`, mockDownloadURL) mockDownloadURLResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(downloadURLBody)), + Body: io.NopCloser(strings.NewReader(downloadURLBody)), } mockGen3Interface. EXPECT(). @@ -53,7 +53,7 @@ func TestGetDownloadResponse_withShepherd(t *testing.T) { // Mock the request for the file at mockDownloadURL. mockFileResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader("It work")), + Body: io.NopCloser(strings.NewReader("It work")), } mockGen3Interface. EXPECT(). @@ -112,7 +112,7 @@ func TestGetDownloadResponse_noShepherd(t *testing.T) { // Mock the request for the file at mockDownloadURL. mockFileResponse := http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader("It work")), + Body: io.NopCloser(strings.NewReader("It work")), } mockGen3Interface. EXPECT(). @@ -232,7 +232,7 @@ func TestGeneratePresignedURL_withShepherd(t *testing.T) { }`, mockGUID, mockPresignedURL) mockUploadURLResponse := http.Response{ StatusCode: 201, - Body: ioutil.NopCloser(strings.NewReader(presignedURLBody)), + Body: io.NopCloser(strings.NewReader(presignedURLBody)), } mockGen3Interface. EXPECT(). From 5bdf30c48a7cbbaeb3a93cf1887dab1dc3c67e16 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 31 Jul 2025 11:06:40 -0700 Subject: [PATCH 6/9] feedbacks, fix tests --- data-client/.DS_Store | Bin 6148 -> 0 bytes data-client/commonUtils/commonUtils.go | 2 +- data-client/g3cmd/download-multiple.go | 7 +++++-- data-client/g3cmd/utils.go | 14 +++++++------- data-client/jwt/configure.go | 23 ++++++++--------------- data-client/jwt/functions.go | 4 ++-- data-client/jwt/update.go | 7 +++++-- data-client/mocks/mock_gen3interface.go | 2 +- tests/functions_test.go | 8 ++++---- 9 files changed, 33 insertions(+), 34 deletions(-) delete mode 100644 data-client/.DS_Store diff --git a/data-client/.DS_Store b/data-client/.DS_Store deleted file mode 100644 index 4bc48646067ad752948dbf15c23b2cb12a4cd324..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z<+|Z7D(y3Oz1(t=M3xh?h|73mDOZN=;0u!8BW%)Er77XMG``#OHBl zcLOa3k0N#kcE8#A+0A^A{b7vp!6L93vlwFyXowt@5)-$3&!R9%yWLN|Q>oT=>maTAgZnHqH}%p*GVtbiIJ%ZH@(bPb zZ^Cglvi2`zoO)qAo~wi~7(>YYZ5RhK8^}c*Br4a{jwZ;+YA=`le!nA*>{UlBZTr;j zh_haAwQ4pF4v){TMlaDcmT#I!4vZ_=Gg!boC~FnHy7M@a@dT_zMiEI!3=jjv05Pzg z44Ctv(b~@PscK??82Et!+#duqM8{yRQEeU2;q@8)RYVlf@hyQU3_1pLjo<;{Iu%f- za`VLCIvwo7#5o3YjXIriwKB|ORxTbdT&)gvp~4w=G*V9t5CfYGRP@lo^ZyclnbJr8 zW(tjn0b<~vF~F-scQ}Be%-Q;_JUnX!v`1(t7?-000{X@!00y{^w3So)1?mvz7|b=| TENEBhfOHX1giuEe`~m}Ce#S~d diff --git a/data-client/commonUtils/commonUtils.go b/data-client/commonUtils/commonUtils.go index 908825d..2737305 100644 --- a/data-client/commonUtils/commonUtils.go +++ b/data-client/commonUtils/commonUtils.go @@ -125,7 +125,7 @@ func ParseRootPath(filePath string) (string, error) { func GetAbsolutePath(filePath string) (string, error) { fullFilePath, err := ParseRootPath(filePath) if err != nil { - return fullFilePath, err + return "", err } fullFilePath, err = filepath.Abs(fullFilePath) return fullFilePath, err diff --git a/data-client/g3cmd/download-multiple.go b/data-client/g3cmd/download-multiple.go index b1faeaa..e432289 100644 --- a/data-client/g3cmd/download-multiple.go +++ b/data-client/g3cmd/download-multiple.go @@ -68,7 +68,7 @@ func AskGen3ForFileInfo(gen3Interface Gen3Interface, guid string, protocol strin } else { // Attempt to get the filename from Indexd endPointPostfix := commonUtils.IndexdIndexEndpoint + "/" + guid - indexdMsg, err := gen3Interface.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "", nil) + indexdMsg, err := gen3Interface.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "", nil) if err != nil { log.Println("Error occurred when querying filename from IndexD: " + err.Error()) log.Println("Using GUID for filename instead.") @@ -301,7 +301,10 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat fmt.Println("NOTICE: flag \"rename\" only works if flag \"filename-format\" is \"original\"") rename = false } - validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt) + err = validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt) + if err != nil { + return err + } protocolText := "" if protocol != "" { diff --git a/data-client/g3cmd/utils.go b/data-client/g3cmd/utils.go index 77fa7d7..e545ce5 100644 --- a/data-client/g3cmd/utils.go +++ b/data-client/g3cmd/utils.go @@ -130,7 +130,7 @@ func InitMultipartUpload(g3 Gen3Interface, filename string, bucketName string) ( return "", "", errors.New("Error has occurred during marshalling data for multipart upload initialization, detailed error message: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartInitEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartInitEndpoint, "application/json", objectBytes) if err != nil { if strings.Contains(err.Error(), "404") { @@ -159,7 +159,7 @@ func GenerateMultipartPresignedURL(g3 Gen3Interface, key string, uploadID string return "", errors.New("Error has occurred during marshalling data for multipart upload presigned url generation, detailed error message: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) if err != nil { return "", errors.New("Error has occurred during multipart upload presigned url generation, detailed error message: " + err.Error()) @@ -185,7 +185,7 @@ func CompleteMultipartUpload(g3 Gen3Interface, key string, uploadID string, part return errors.New("Error has occurred during marshalling data for multipart upload, detailed error message: " + err.Error()) } - _, err = g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) + _, err = g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) if err != nil { return errors.New("Error has occurred during completing multipart upload, detailed error message: " + err.Error()) } @@ -228,7 +228,7 @@ func GetDownloadResponse(g3 Gen3Interface, fdrObject *commonUtils.FileDownloadRe } } else { endPointPostfix := commonUtils.FenceDataDownloadEndpoint + "/" + fdrObject.GUID + protocolText - msg, err := g3.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "", nil) + msg, err := g3.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "", nil) if err != nil || msg.URL == "" { errorMsg := "Error occurred when getting download URL for object " + fdrObject.GUID @@ -334,7 +334,7 @@ func GeneratePresignedURL(g3 Gen3Interface, filename string, fileMetadata common if err != nil { return "", "", errors.New("Error occurred when marshalling object: " + err.Error()) } - msg, err := g3.DoRequestWithSignedHeader(profileConfig, commonUtils.FenceDataUploadEndpoint, "application/json", objectBytes) + msg, err := g3.DoRequestWithSignedHeader(&profileConfig, commonUtils.FenceDataUploadEndpoint, "application/json", objectBytes) if err != nil { return "", "", errors.New("Something went wrong. Maybe you don't have permission to upload data or Fence is misconfigured. Detailed error message: " + err.Error()) @@ -355,7 +355,7 @@ func GenerateUploadRequest(g3 Gen3Interface, furObject commonUtils.FileUploadReq endPointPostfix += "&bucket=" + furObject.Bucket } - msg, err := g3.DoRequestWithSignedHeader(profileConfig, endPointPostfix, "application/json", nil) + msg, err := g3.DoRequestWithSignedHeader(&profileConfig, endPointPostfix, "application/json", nil) if err != nil && !strings.Contains(err.Error(), "No GUID found") { return furObject, errors.New("Upload error: " + err.Error()) } @@ -718,7 +718,7 @@ type Gen3Interface interface { CheckPrivileges(profileConfig *jwt.Credential) (string, map[string]interface{}, error) CheckForShepherdAPI(profileConfig *jwt.Credential) (bool, error) GetResponse(profileConfig *jwt.Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(profileConfig jwt.Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) + DoRequestWithSignedHeader(profileConfig *jwt.Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) GetHost(profileConfig *jwt.Credential) (*url.URL, error) } diff --git a/data-client/jwt/configure.go b/data-client/jwt/configure.go index 202763d..8116d5e 100644 --- a/data-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -18,6 +18,8 @@ import ( "gopkg.in/ini.v1" ) +var ErrProfileNotFound = errors.New("profile not found in config file") + type Credential struct { Profile string KeyId string @@ -157,7 +159,8 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { cfg.Section(profileConfig.Profile).Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) err = cfg.SaveTo(configPath) if err != nil { - log.Println("error occurred when saving config file: " + err.Error()) + errs := fmt.Errorf("error occurred when saving config file: " + err.Error()) + return errs } return nil } @@ -207,8 +210,7 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { homeDir, err := homedir.Dir() if err != nil { errs := fmt.Errorf("Error occurred when getting home directory: " + err.Error()) - log.Println(errs.Error()) - return Credential{}, err + return Credential{}, errs } configPath := path.Join(homeDir + commonUtils.PathSeparator + ".gen3" + commonUtils.PathSeparator + "gen3_client_config.ini") profileConfig := Credential{ @@ -219,48 +221,39 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { APIEndpoint: "", } if _, err := os.Stat(configPath); os.IsNotExist(err) { - log.Println("No config file found in ~/.gen3/") - fmt.Println("Run configure command (with a profile if desired) to set up account credentials \n" + - "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org") - return Credential{}, fmt.Errorf("No config file found in ~/.gen3/") + return Credential{}, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ + "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) } // If profile not in config file, prompt user to set up config first cfg, err := ini.Load(configPath) if err != nil { errs := fmt.Errorf("Error occurred when reading config file: " + err.Error()) - log.Println(errs) return Credential{}, errs } sec, err := cfg.GetSection(profile) if err != nil { - errs := fmt.Errorf("Profile not in config file. Need to run \"data-client configure --profile=" + profile + " --cred= --apiendpoint=\" first") - fmt.Println(errs.Error()) - return Credential{}, err + return Credential{}, fmt.Errorf("%w: Need to run \"data-client configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) } // Read in API key, key ID and endpoint for given profile profileConfig.KeyId = sec.Key("key_id").String() if profileConfig.KeyId == "" { errs := fmt.Errorf("key_id not found in profile.") - log.Println(errs) return Credential{}, errs } profileConfig.APIKey = sec.Key("api_key").String() if profileConfig.APIKey == "" { errs := fmt.Errorf("api_key not found in profile.") - log.Println(errs) return Credential{}, errs } profileConfig.AccessToken = sec.Key("access_token").String() if profileConfig.AccessToken == "" { errs := fmt.Errorf("access_token not found in profile.") - log.Println(errs) return Credential{}, errs } profileConfig.APIEndpoint = sec.Key("api_endpoint").String() if profileConfig.APIEndpoint == "" { errs := fmt.Errorf("api_endpoint not found in profile.") - log.Println(errs) return Credential{}, errs } // UseShepherd and MinShepherdVersion are optional diff --git a/data-client/jwt/functions.go b/data-client/jwt/functions.go index 54a7b70..56adb24 100644 --- a/data-client/jwt/functions.go +++ b/data-client/jwt/functions.go @@ -254,7 +254,7 @@ func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { return host, nil } -func (f *Functions) DoRequestWithSignedHeader(profileConfig Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { +func (f *Functions) DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { /* Do request with signed header. User may have more than one profile and use a profile to make a request */ @@ -266,7 +266,7 @@ func (f *Functions) DoRequestWithSignedHeader(profileConfig Credential, endpoint method = "POST" } - _, resp, err := f.GetResponse(&profileConfig, endpointPostPrefix, method, contentType, bodyBytes) + _, resp, err := f.GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes) if err != nil { return msg, err } diff --git a/data-client/jwt/update.go b/data-client/jwt/update.go index 16650d4..4c32323 100644 --- a/data-client/jwt/update.go +++ b/data-client/jwt/update.go @@ -5,9 +5,9 @@ import ( "log" "strings" - "github.com/hashicorp/go-version" "github.com/calypr/data-client/data-client/commonUtils" "github.com/calypr/data-client/data-client/logs" + "github.com/hashicorp/go-version" ) func UpdateConfig(profile string, apiEndpoint string, credFile string, useShepherd string, minShepherdVersion string) error { @@ -55,7 +55,10 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe profileConfig.MinShepherdVersion = minShepherdVersion // Store user info in ~/.gen3/gen3_client_config.ini - conf.UpdateConfigFile(*profileConfig) + err = conf.UpdateConfigFile(*profileConfig) + if err != nil { + return err + } log.Println(`Profile '` + profile + `' has been configured successfully.`) err = logs.CloseMessageLog() if err != nil { diff --git a/data-client/mocks/mock_gen3interface.go b/data-client/mocks/mock_gen3interface.go index aed5c01..70f02d2 100644 --- a/data-client/mocks/mock_gen3interface.go +++ b/data-client/mocks/mock_gen3interface.go @@ -64,7 +64,7 @@ func (mr *MockGen3InterfaceMockRecorder) CheckPrivileges(arg0 interface{}) *gomo } // DoRequestWithSignedHeader mocks base method -func (m *MockGen3Interface) DoRequestWithSignedHeader(arg0 jwt.Credential, arg1, arg2 string, arg3 []byte) (jwt.JsonMessage, error) { +func (m *MockGen3Interface) DoRequestWithSignedHeader(arg0 *jwt.Credential, arg1, arg2 string, arg3 []byte) (jwt.JsonMessage, error) { ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(jwt.JsonMessage) ret1, _ := ret[1].(error) diff --git a/tests/functions_test.go b/tests/functions_test.go index 70c64e6..3aa0609 100755 --- a/tests/functions_test.go +++ b/tests/functions_test.go @@ -24,7 +24,7 @@ func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) if err == nil { t.Fail() @@ -47,7 +47,7 @@ func TestDoRequestWithSignedHeaderGoodToken(t *testing.T) { mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/data/download/test_uuid", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil { t.Fail() @@ -73,7 +73,7 @@ func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil { t.Fail() @@ -99,7 +99,7 @@ func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(2) - _, err := testFunction.DoRequestWithSignedHeader(profileConfig, "/user/data/download/test_uuid", "", nil) + _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) if err != nil && !strings.Contains(err.Error(), "401") { t.Fail() From 61dadd74e84dada87fab61f10292580c9fb05d1c Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 31 Jul 2025 11:17:21 -0700 Subject: [PATCH 7/9] fix lint errors --- data-client/g3cmd/download-multiple.go | 10 ++++++---- data-client/g3cmd/upload-multiple.go | 2 +- data-client/g3cmd/upload-single.go | 8 ++++---- data-client/g3cmd/upload.go | 2 +- data-client/g3cmd/utils.go | 2 +- data-client/jwt/configure.go | 6 +++--- data-client/jwt/functions.go | 2 +- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/data-client/g3cmd/download-multiple.go b/data-client/g3cmd/download-multiple.go index e432289..2db3a6f 100644 --- a/data-client/g3cmd/download-multiple.go +++ b/data-client/g3cmd/download-multiple.go @@ -433,8 +433,7 @@ func init() { manifestBytes, err := io.ReadAll(manifestFileReader) if err != nil { - log.Printf("Failed reading manifest %s, %v\n", manifestPath, err) - return + log.Fatalf("Failed reading manifest %s, %v\n", manifestPath, err) } manifestFileBar.Finish() @@ -444,10 +443,13 @@ func init() { log.Fatalf("Error has occurred during unmarshalling manifest object: %v\n", err) } - downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) + err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) + if err != nil { + log.Fatalf(err.Error()) + } err = logs.CloseMessageLog() if err != nil { - log.Println(err.Error()) + log.Fatalf(err.Error()) } }, } diff --git a/data-client/g3cmd/upload-multiple.go b/data-client/g3cmd/upload-multiple.go index b74f5f3..01c84cf 100644 --- a/data-client/g3cmd/upload-multiple.go +++ b/data-client/g3cmd/upload-multiple.go @@ -79,7 +79,7 @@ func init() { uploadPath, err := commonUtils.GetAbsolutePath(uploadPath) if err != nil { - log.Fatalf("Error when parsing file paths: " + err.Error()) + log.Fatalf("Error when parsing file paths: %s", err.Error()) } filePaths := make([]string, 0) diff --git a/data-client/g3cmd/upload-single.go b/data-client/g3cmd/upload-single.go index 63d7bfc..653ce1d 100644 --- a/data-client/g3cmd/upload-single.go +++ b/data-client/g3cmd/upload-single.go @@ -8,8 +8,8 @@ import ( "os" "path/filepath" - "github.com/spf13/cobra" "github.com/calypr/data-client/data-client/commonUtils" + "github.com/spf13/cobra" ) func init() { @@ -126,9 +126,9 @@ func UploadSingle(profile string, guid string, filePath string, bucketName strin log.Println(errStr.Error()) return errStr // logs.IncrementScore(logs.ScoreBoardLen - 1) // update failed score - } else { - // logs.IncrementScore(0) // update succeeded score - } + } /*else { + // logs.IncrementScore(0) // update succeeded score + }*/ // logs.PrintScoreBoard() // logs.CloseAll() log.Println("INSIDE: upload complete") diff --git a/data-client/g3cmd/upload.go b/data-client/g3cmd/upload.go index dde4723..a691725 100644 --- a/data-client/g3cmd/upload.go +++ b/data-client/g3cmd/upload.go @@ -60,7 +60,7 @@ func init() { uploadPath, _ = commonUtils.GetAbsolutePath(uploadPath) filePaths, err := commonUtils.ParseFilePaths(uploadPath, hasMetadata) if err != nil { - log.Fatalf("Error when parsing file paths: " + err.Error()) + log.Fatalf("Error when parsing file paths: %s", err.Error()) } if len(filePaths) == 0 { log.Println("No file has been found in the provided location \"" + uploadPath + "\"") diff --git a/data-client/g3cmd/utils.go b/data-client/g3cmd/utils.go index e545ce5..3c3d8cd 100644 --- a/data-client/g3cmd/utils.go +++ b/data-client/g3cmd/utils.go @@ -24,7 +24,7 @@ import ( pb "gopkg.in/cheggaaa/pb.v1" ) -// go:generate mockgen -destination=./data-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/data-client/g3cmd Gen3Interface +//go:generate mockgen -destination=./data-client/mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/data-client/g3cmd Gen3Interface // ManifestObject represents an object from manifest that downloaded from windmill / data-portal type ManifestObject struct { diff --git a/data-client/jwt/configure.go b/data-client/jwt/configure.go index 8116d5e..6ff07e6 100644 --- a/data-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -85,7 +85,7 @@ func (conf *Configure) ReadCredentials(filePath string) (*Credential, error) { jsonContent = strings.Replace(jsonContent, "api_key", "APIKey", -1) err := json.Unmarshal([]byte(jsonContent), &profileConfig) if err != nil { - errs := fmt.Errorf("Cannot read json file: " + err.Error()) + errs := fmt.Errorf("Cannot read json file: %s", err.Error()) log.Println(errs.Error()) return nil, errs } @@ -141,13 +141,13 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { */ configPath, err := conf.GetConfigPath() if err != nil { - errs := fmt.Errorf("error occurred when getting config path: " + err.Error()) + errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) log.Println(errs.Error()) return errs } cfg, err := ini.Load(configPath) if err != nil { - errs := fmt.Errorf("error occurred when loading config file: " + err.Error()) + errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) log.Println(errs.Error()) return errs } diff --git a/data-client/jwt/functions.go b/data-client/jwt/functions.go index 56adb24..3d7cdff 100644 --- a/data-client/jwt/functions.go +++ b/data-client/jwt/functions.go @@ -207,7 +207,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st var err error if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, errors.New(fmt.Sprintf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig)) + return "", resp, fmt.Errorf(fmt.Sprintf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig)) } host, _ := url.Parse(profileConfig.APIEndpoint) prefixEndPoint := host.Scheme + "://" + host.Host From 5de12f74e4fc760b5fb24ed6d6e3b960cc1187cc Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 31 Jul 2025 11:29:48 -0700 Subject: [PATCH 8/9] fix lint --- data-client/g3cmd/download-multiple.go | 6 +++--- data-client/g3cmd/download-single.go | 7 +++++-- data-client/g3cmd/upload-multiple.go | 5 ++++- data-client/g3cmd/upload-single.go | 5 ++++- data-client/jwt/configure.go | 6 +++--- data-client/jwt/update.go | 2 +- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/data-client/g3cmd/download-multiple.go b/data-client/g3cmd/download-multiple.go index 2db3a6f..0b42213 100644 --- a/data-client/g3cmd/download-multiple.go +++ b/data-client/g3cmd/download-multiple.go @@ -313,7 +313,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat err = os.MkdirAll(downloadPath, 0766) if err != nil { - return fmt.Errorf("Cannot create folder \"" + downloadPath + "\"") + return fmt.Errorf("Cannot create folder %s", downloadPath) } renamedFiles := make([]RenamedOrSkippedFileInfo, 0) @@ -445,11 +445,11 @@ func init() { err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) if err != nil { - log.Fatalf(err.Error()) + log.Fatalln(err.Error()) } err = logs.CloseMessageLog() if err != nil { - log.Fatalf(err.Error()) + log.Fatalln(err.Error()) } }, } diff --git a/data-client/g3cmd/download-single.go b/data-client/g3cmd/download-single.go index af5ffc2..0a1b1ba 100644 --- a/data-client/g3cmd/download-single.go +++ b/data-client/g3cmd/download-single.go @@ -3,8 +3,8 @@ package g3cmd import ( "log" - "github.com/spf13/cobra" "github.com/calypr/data-client/data-client/logs" + "github.com/spf13/cobra" ) func init() { @@ -34,7 +34,10 @@ func init() { ObjectID: guid, } objects := []ManifestObject{obj} - downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) + err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) + if err != nil { + log.Println(err.Error()) + } err = logs.CloseMessageLog() if err != nil { log.Println(err.Error()) diff --git a/data-client/g3cmd/upload-multiple.go b/data-client/g3cmd/upload-multiple.go index 01c84cf..62584bb 100644 --- a/data-client/g3cmd/upload-multiple.go +++ b/data-client/g3cmd/upload-multiple.go @@ -129,7 +129,10 @@ func init() { processSingleUploads(gen3Interface, singlePartFilePaths, bucketName, includeSubDirName, uploadPath) } if len(multipartFilePaths) > 0 { - processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + err := processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + if err != nil { + log.Fatalln(err.Error()) + } } if !logs.IsFailedLogMapEmpty() { retryUpload(logs.GetFailedLogMap()) diff --git a/data-client/g3cmd/upload-single.go b/data-client/g3cmd/upload-single.go index 653ce1d..c4297ca 100644 --- a/data-client/g3cmd/upload-single.go +++ b/data-client/g3cmd/upload-single.go @@ -24,7 +24,10 @@ func init() { Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, Run: func(cmd *cobra.Command, args []string) { fmt.Printf("Notice: this is the upload method which requires the user to provide a GUID. In this method file will be uploaded to a specified GUID.\nIf your intention is to upload file without pre-existing GUID, consider to use \"./data-client upload\" instead.\n\n") - UploadSingle(profile, guid, filePath, bucketName) + err := UploadSingle(profile, guid, filePath, bucketName) + if err != nil { + log.Fatalln(err.Error()) + } }, } uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") diff --git a/data-client/jwt/configure.go b/data-client/jwt/configure.go index 6ff07e6..067e510 100644 --- a/data-client/jwt/configure.go +++ b/data-client/jwt/configure.go @@ -159,7 +159,7 @@ func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { cfg.Section(profileConfig.Profile).Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) err = cfg.SaveTo(configPath) if err != nil { - errs := fmt.Errorf("error occurred when saving config file: " + err.Error()) + errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) return errs } return nil @@ -209,7 +209,7 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { homeDir, err := homedir.Dir() if err != nil { - errs := fmt.Errorf("Error occurred when getting home directory: " + err.Error()) + errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) return Credential{}, errs } configPath := path.Join(homeDir + commonUtils.PathSeparator + ".gen3" + commonUtils.PathSeparator + "gen3_client_config.ini") @@ -228,7 +228,7 @@ func (conf *Configure) ParseConfig(profile string) (Credential, error) { // If profile not in config file, prompt user to set up config first cfg, err := ini.Load(configPath) if err != nil { - errs := fmt.Errorf("Error occurred when reading config file: " + err.Error()) + errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) return Credential{}, errs } sec, err := cfg.GetSection(profile) diff --git a/data-client/jwt/update.go b/data-client/jwt/update.go index 4c32323..1cd6153 100644 --- a/data-client/jwt/update.go +++ b/data-client/jwt/update.go @@ -49,7 +49,7 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe if minShepherdVersion != "" { _, err = version.NewVersion(minShepherdVersion) if err != nil { - return fmt.Errorf("Error occurred when validating minShepherdVersion: " + err.Error()) + return fmt.Errorf("Error occurred when validating minShepherdVersion: %s", err.Error()) } } profileConfig.MinShepherdVersion = minShepherdVersion From 085b5d0f22f09a6d946fc0a90197f5357312f9f9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 31 Jul 2025 11:36:38 -0700 Subject: [PATCH 9/9] fix lint errors --- data-client/g3cmd/upload.go | 5 ++++- data-client/jwt/functions.go | 2 +- data-client/jwt/update.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/data-client/g3cmd/upload.go b/data-client/g3cmd/upload.go index a691725..6c22424 100644 --- a/data-client/g3cmd/upload.go +++ b/data-client/g3cmd/upload.go @@ -159,7 +159,10 @@ func init() { if len(multipartFilePaths) > 0 { // NOTE(@mpingram) - For the moment Shepherd doesn't support multipart uploads. // Throw an error if Shepherd is enabled and user attempts to multipart upload. - processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + err := processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath) + if err != nil { + log.Println(err.Error()) + } } if !logs.IsFailedLogMapEmpty() { diff --git a/data-client/jwt/functions.go b/data-client/jwt/functions.go index 3d7cdff..7184f22 100644 --- a/data-client/jwt/functions.go +++ b/data-client/jwt/functions.go @@ -207,7 +207,7 @@ func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix st var err error if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, fmt.Errorf(fmt.Sprintf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig)) + return "", resp, fmt.Errorf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig) } host, _ := url.Parse(profileConfig.APIEndpoint) prefixEndPoint := host.Scheme + "://" + host.Host diff --git a/data-client/jwt/update.go b/data-client/jwt/update.go index 1cd6153..65e8826 100644 --- a/data-client/jwt/update.go +++ b/data-client/jwt/update.go @@ -26,7 +26,7 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } parsedURL, err := conf.ValidateUrl(apiEndpoint) if err != nil { - return fmt.Errorf("Errr occurred when validating apiendpoint URL: " + err.Error()) + return fmt.Errorf("Errr occurred when validating apiendpoint URL: %s", err.Error()) } prefixEndPoint := parsedURL.Scheme + "://" + parsedURL.Host @@ -39,7 +39,7 @@ func UpdateConfig(profile string, apiEndpoint string, credFile string, useShephe } else if strings.Contains(receivedErrorString, "404") || strings.Contains(receivedErrorString, "405") || strings.Contains(receivedErrorString, "no such host") { errorMessageString = `The provided apiendpoint '` + prefixEndPoint + `' is possibly not a valid Gen3 data commons` } - return fmt.Errorf("Error occurred when validating profile config: " + errorMessageString) + return fmt.Errorf("Error occurred when validating profile config: %s", errorMessageString) } profileConfig.APIEndpoint = apiEndpoint