Fix/logging

Dockerfile

@@ -4,7 +4,7 @@ ENV CGO_ENABLED=0
 ENV GOOS=linux
 ENV GOARCH=amd64
 
-WORKDIR $GOPATH/src/github.com/uc-cdis/gen3-client/
+WORKDIR $GOPATH/src/github.com/calypr/data-client/
 
 COPY go.mod .
 COPY go.sum .
@@ -20,10 +20,10 @@ RUN COMMIT=$(git rev-parse HEAD); \
     'const ('\
     '    gitcommit="'"${COMMIT}"'"'\
     '    gitversion="'"${VERSION}"'"'\
-    ')' > gen3-client/g3cmd/gitversion.go \
-    && go build -o /gen3-client
+    ')' > data-client/g3cmd/gitversion.go \
+    && go build -o /data-client
 
 FROM scratch
 COPY --from=build-deps /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build-deps /gen3-client /gen3-client
-CMD ["/gen3-client"]
+COPY --from=build-deps /data-client /data-client
+CMD ["/data-client"]

README.md

@@ -1,24 +1,24 @@
-# gen3-client
+# data-client
 [![Build Status](https://travis-ci.org/uc-cdis/cdis-data-client.svg?branch=master)](https://travis-ci.org/uc-cdis/cdis-data-client)
 [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/uc-cdis/cdis-data-client?sort=semver)](https://github.com/uc-cdis/cdis-data-client/releases)
 
-`gen3-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons.
+`data-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons.
 
-Read more about what it does and how to use it in the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/).
+Read more about what it does and how to use it in the `data-client` [user guide](https://gen3.org/resources/user/data-client/).
 
-`gen3-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra).
+`data-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra).
 
 
 ## Installation
 
-(The following instruction is for compiling and installing the `gen3-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases))
+(The following instruction is for compiling and installing the `data-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases))
 
 First, [install Go and the Go tools](https://golang.org/doc/install) if you have not already done so. [Set up your workspace and your GOPATH.](https://golang.org/doc/code.html)
 
 
 Then:
 ```
-go get -d github.com/uc-cdis/gen3-client
+go get -d github.com/calypr/data-client
 go install
 ```
 
@@ -30,43 +30,43 @@ go install
 mkdir -p $GOPATH/src/github.com/uc-cdis
 cd $GOPATH/src/github.com/uc-cdis
 git clone git@github.com:uc-cdis/cdis-data-client.git
-mv cdis-data-client gen3-client
-cd gen3-client
+mv cdis-data-client data-client
+cd data-client
 go get -d ./...
 go install .
 ```
 
-Now you should have `gen3-client` successfully installed. For a comprehensive instruction on how to configure and use `gen3-client` for uploading / downloading object files, please refer to the `gen3-client` [user guide](https://gen3.org/resources/user/gen3-client/).
+Now you should have `data-client` successfully installed. For a comprehensive instruction on how to configure and use `data-client` for uploading / downloading object files, please refer to the `data-client` [user guide](https://gen3.org/resources/user/data-client/).
 
 ## Enabling New Gen3 Object Management API
 Some Gen3 data commons support uploading files through the new Gen3 Object Management API.
 
 > NOTE: The service powering this API is sometimes referred to as our object "Shepherd"
 
-To enable gen3-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `gen3-client configure`, e.g.:
+To enable data-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `data-client configure`, e.g.:
 ```
-$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true
+$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true
 ```
-If this flag is set, the gen3-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure.
+If this flag is set, the data-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure.
 
 
 >You may also need to configure the version of the Gen3 Object Management API that the client will interact with. This is set to a default of Gen3 Object Management API `v2.0.0`, but can
->be raised or lowered by passing the `min-shepherd-version` flag to `gen3-client configure`, e.g.:
+>be raised or lowered by passing the `min-shepherd-version` flag to `data-client configure`, e.g.:
 >```
->$ gen3-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0
+>$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0
 >```
 
 ### Uploading Additional File Object Metadata to Gen3 Object Management API
-The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files. 
+The Gen3 Object Management API supports uploading additional *public access* file object metadata when uploading data files.
 
 > WARNING: Additional File Object Metadata is exposed publically and thus should not be controlled/sensitive data
 
-You can upload file metadata using the `gen3-client upload` command with the  `--metadata` flag. E.g.:
+You can upload file metadata using the `data-client upload` command with the  `--metadata` flag. E.g.:
 ```
-gen3-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata
+data-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata
 ```
 
-This will tell `gen3-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`.
+This will tell `data-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`.
 A metadata file should be located in the same folder as the file to be uploaded, and should be named `[filename]_metadata.json`.
 
 The metadata file should be a JSON file in the format:

gen3-client/commonUtils/commonUtils.go → data-client/commonUtils/commonUtils.go

@@ -17,13 +17,13 @@ import (
 
 // DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API
 // endpoints if available.
-// The user can override this default using the `gen3-client configure` command.
+// The user can override this default using the `data-client configure` command.
 const DefaultUseShepherd = false
 
 // DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use.
 // Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is
 // below this number the gen3client will instead warn the user and fall back to fence/indexd.
-// The user can override this default using the `gen3-client configure` command.
+// The user can override this default using the `data-client configure` command.
 const DefaultMinShepherdVersion = "2.0.0"
 
 // ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API
@@ -74,7 +74,7 @@ type FileUploadRequestObject struct {
 	PresignedURL string
 	Request      *http.Request
 	Bar          *pb.ProgressBar
-	Bucket 	 	 string `json:"bucket,omitempty"`
+	Bucket       string `json:"bucket,omitempty"`
 }
 
 // FileDownloadResponseObject defines a object for file download
@@ -106,25 +106,28 @@ type RetryObject struct {
 	GUID         string
 	RetryCount   int
 	Multipart    bool
-	Bucket 		 string
+	Bucket       string
 }
 
 // ParseRootPath parses dirname that has "~" in the beginning
-func ParseRootPath(filePath string) string {
+func ParseRootPath(filePath string) (string, error) {
 	if filePath != "" && filePath[0] == '~' {
 		homeDir, err := homedir.Dir()
 		if err != nil {
-			log.Fatalln(err)
+			return "", err
 		}
-		return homeDir + filePath[1:]
+		return homeDir + filePath[1:], nil
 	}
-	return filePath
+	return filePath, nil
 }
 
 // GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning
 func GetAbsolutePath(filePath string) (string, error) {
-	fullFilePath := ParseRootPath(filePath)
-	fullFilePath, err := filepath.Abs(fullFilePath)
+	fullFilePath, err := ParseRootPath(filePath)
+	if err != nil {
+		return "", err
+	}
+	fullFilePath, err = filepath.Abs(fullFilePath)
 	return fullFilePath, err
 }
 

gen3-client/g3cmd/auth.go → data-client/g3cmd/auth.go

@@ -6,7 +6,7 @@ import (
 	"sort"
 
 	"github.com/spf13/cobra"
-	"github.com/uc-cdis/gen3-client/gen3-client/logs"
+	"github.com/calypr/data-client/data-client/logs"
 )
 
 func init() {
@@ -15,12 +15,15 @@ func init() {
 		Use:     "auth",
 		Short:   "Return resource access privileges from profile",
 		Long:    `Gets resource access privileges for specified profile.`,
-		Example: `./gen3-client auth --profile=<profile-name>`,
+		Example: `./data-client auth --profile=<profile-name>`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// don't initialize transmission logs for non-uploading related commands
 			logs.SetToBoth()
 			gen3Interface := NewGen3Interface()
-			profileConfig = conf.ParseConfig(profile)
+				profileConfig, err := conf.ParseConfig(profile)
+			if err != nil {
+				log.Fatalf("Fatal config parse error: %s\n", err)
+			}
 
 			host, resourceAccess, err := gen3Interface.CheckPrivileges(&profileConfig)
 

gen3-client/g3cmd/configure.go → data-client/g3cmd/configure.go

@@ -2,11 +2,12 @@ package g3cmd
 
 import (
 	"fmt"
+	"log"
 
+	"github.com/calypr/data-client/data-client/commonUtils"
+	"github.com/calypr/data-client/data-client/jwt"
+	"github.com/calypr/data-client/data-client/logs"
 	"github.com/spf13/cobra"
-	"github.com/uc-cdis/gen3-client/gen3-client/commonUtils"
-	"github.com/uc-cdis/gen3-client/gen3-client/jwt"
-	"github.com/uc-cdis/gen3-client/gen3-client/logs"
 )
 
 var conf jwt.Configure // Why is this a global variable?
@@ -21,12 +22,15 @@ func init() {
 		Short: "Add or modify a configuration profile to your config file",
 		Long: `Configuration file located at ~/.gen3/gen3_client_config.ini
 	If a field is left empty, the existing value (if it exists) will remain unchanged`,
-		Example: `./gen3-client configure --profile=<profile-name> --cred=<path-to-credential/cred.json> --apiendpoint=https://data.mycommons.org`,
+		Example: `./data-client configure --profile=<profile-name> --cred=<path-to-credential/cred.json> --apiendpoint=https://data.mycommons.org`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// don't initialize transmission logs for non-uploading related commands
 			logs.SetToBoth()
 
-			jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion)
+			err := jwt.UpdateConfig(profile, apiEndpoint, credFile, useShepherd, minShepherdVersion)
+			if err != nil {
+				log.Println(err.Error())
+			}
 
 		},
 	}

gen3-client/g3cmd/delete.go → data-client/g3cmd/delete.go

@@ -13,8 +13,8 @@ var deleteCmd = &cobra.Command{ // nolint:deadcode,unused,varcheck
 	Short: "Send DELETE HTTP Request for given URI",
 	Long: `Deletes a given URI from the database.
 If no profile is specified, "default" profile is used for authentication.`,
-	Example: `./gen3-client delete --uri=v0/submission/bpa/test/entities/example_id
-	  ./gen3-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`,
+	Example: `./data-client delete --uri=v0/submission/bpa/test/entities/example_id
+	  ./data-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`,
 	Run: func(cmd *cobra.Command, args []string) {
 		log.Fatalf("Not supported!")
 		// request := new(jwt.Request)

gen3-client/g3cmd/download-multiple.go → data-client/g3cmd/download-multiple.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -14,8 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/uc-cdis/gen3-client/gen3-client/commonUtils"
-	"github.com/uc-cdis/gen3-client/gen3-client/logs"
+	"github.com/calypr/data-client/data-client/commonUtils"
+	"github.com/calypr/data-client/data-client/logs"
 	pb "gopkg.in/cheggaaa/pb.v1"
 
 	"github.com/spf13/cobra"
@@ -158,9 +157,9 @@ func processOriginalFilename(downloadPath string, actualFilename string) string
 	}
 }
 
-func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) {
+func validateFilenameFormat(downloadPath string, filenameFormat string, rename bool, noPrompt bool) error {
 	if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" {
-		log.Fatalln("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only")
+		return fmt.Errorf("Invalid option found! Option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only")
 	}
 	if filenameFormat == "guid" || filenameFormat == "combined" {
 		fmt.Printf("WARNING: in \"guid\" or \"combined\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath)
@@ -177,6 +176,7 @@ func validateFilenameFormat(downloadPath string, filenameFormat string, rename b
 	} else {
 		fmt.Printf("NOTICE: flag \"rename\" was set to true in \"original\" mode, duplicated files under \"%s\" will be renamed by appending a counter value to the original filenames\n", downloadPath)
 	}
+	return nil
 }
 
 func validateLocalFileStat(downloadPath string, filename string, filesize int64, skipCompleted bool) commonUtils.FileDownloadResponseObject {
@@ -284,12 +284,15 @@ func batchDownload(g3 Gen3Interface, batchFDRSlice []commonUtils.FileDownloadRes
 	return succeeded
 }
 
-func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) {
+func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) error {
 	if numParallel < 1 {
-		log.Fatalln("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.")
+		return fmt.Errorf("Invalid value for option \"numparallel\": must be a positive integer! Please check your input.")
 	}
 
-	downloadPath = commonUtils.ParseRootPath(downloadPath)
+	downloadPath, err := commonUtils.ParseRootPath(downloadPath)
+	if err != nil {
+		return fmt.Errorf("downloadFile Error: %s", err.Error())
+	}
 	if !strings.HasSuffix(downloadPath, "/") {
 		downloadPath += "/"
 	}
@@ -298,16 +301,19 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat
 		fmt.Println("NOTICE: flag \"rename\" only works if flag \"filename-format\" is \"original\"")
 		rename = false
 	}
-	validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt)
+	err = validateFilenameFormat(downloadPath, filenameFormat, rename, noPrompt)
+	if err != nil {
+		return err
+	}
 
 	protocolText := ""
 	if protocol != "" {
 		protocolText = "?protocol=" + protocol
 	}
 
-	err := os.MkdirAll(downloadPath, 0766)
+	err = os.MkdirAll(downloadPath, 0766)
 	if err != nil {
-		log.Fatalln("Cannot create folder \"" + downloadPath + "\"")
+		return fmt.Errorf("Cannot create folder %s", downloadPath)
 	}
 
 	renamedFiles := make([]RenamedOrSkippedFileInfo, 0)
@@ -381,6 +387,7 @@ func downloadFile(objects []ManifestObject, downloadPath string, filenameFormat
 			log.Println(err.Error())
 		}
 	}
+	return nil
 }
 
 func init() {
@@ -397,11 +404,15 @@ func init() {
 		Use:     "download-multiple",
 		Short:   "Download multiple of files from a specified manifest",
 		Long:    `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`,
-		Example: `./gen3-client download-multiple --profile=<profile-name> --manifest=<path-to-manifest/manifest.json> --download-path=<path-to-file-dir/>`,
+		Example: `./data-client download-multiple --profile=<profile-name> --manifest=<path-to-manifest/manifest.json> --download-path=<path-to-file-dir/>`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// don't initialize transmission logs for non-uploading related commands
 			logs.SetToBoth()
-			profileConfig = conf.ParseConfig(profile)
+			var err error
+			profileConfig, err = conf.ParseConfig(profile)
+			if err != nil {
+				log.Fatalf("Failed to parse config on profile %s, %v", profile, err)
+			}
 
 			manifestPath, _ = commonUtils.GetAbsolutePath(manifestPath)
 			manifestFile, err := os.Open(manifestPath)
@@ -420,23 +431,25 @@ func init() {
 
 			manifestFileReader := manifestFileBar.NewProxyReader(manifestFile)
 
-			manifestBytes, err := ioutil.ReadAll(manifestFileReader)
-			manifestFileBar.Finish()
-
+			manifestBytes, err := io.ReadAll(manifestFileReader)
 			if err != nil {
-				log.Printf("Failed reading manifest %s, %v\n", manifestPath, err)
-				log.Fatalln("A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal")
+				log.Fatalf("Failed reading manifest %s, %v\n", manifestPath, err)
 			}
+			manifestFileBar.Finish()
+
 			var objects []ManifestObject
 			err = json.Unmarshal(manifestBytes, &objects)
 			if err != nil {
 				log.Fatalf("Error has occurred during unmarshalling manifest object: %v\n", err)
 			}
 
-			downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted)
+			err = downloadFile(objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted)
+			if err != nil {
+				log.Fatalln(err.Error())
+			}
 			err = logs.CloseMessageLog()
 			if err != nil {
-				log.Println(err.Error())
+				log.Fatalln(err.Error())
 			}
 		},
 	}