Add option --retries-on-failure to downloads

Commands `get`, `get-by-version`, `get-unpack`, and `import` now can
set the number of retries on failure.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### NEW FEATURES
 
+* Add option `--retries-on-failure` to downloads `get*` and `import` commands
 * Add file `connection_super_user.conf` and `connection_super_user.json` to each sandbox
 
 ## 1.67.0	10-Aug-2022

cmd/downloads.go

@@ -269,6 +269,7 @@ func getCommonFlags(cmd *cobra.Command) ops.DownloadsOptions {
 	options.ProgressStep, _ = cmd.Flags().GetInt64(globals.ProgressStepLabel)
 	options.VerbosityLevel, _ = cmd.Flags().GetInt(globals.VerbosityLabel)
 	options.Version, _ = cmd.Flags().GetString(globals.UnpackVersionLabel)
+	options.Retries, _ = cmd.Flags().GetInt64(globals.RetriesOnFailure)
 	return options
 }
 
@@ -332,53 +333,56 @@ func resetTarballCollection(cmd *cobra.Command, args []string) {
 	}
 }
 
-func importTarballCollection(cmd *cobra.Command, args []string) {
+func importTarballCollection(cmd *cobra.Command, args []string) error {
 	if len(args) < 1 {
-		common.Exit(1, "command 'import' requires a file name")
+		return fmt.Errorf("command 'import' requires a file name")
 	}
 	fileName := args[0]
 
 	if fileName == "remote-github" || fileName == "remote-tarballs" {
 		fileName = defaults.Defaults().RemoteTarballUrl
 	}
 
+	retries, _ := cmd.Flags().GetInt64(globals.RetriesOnFailure)
+
 	if common.IsUrl(fileName) {
 		fileUrl := fileName
 		re := regexp.MustCompile(`^(http|https)://`)
 		fileName = common.BaseName(re.ReplaceAllString(fileUrl, ""))
 		if common.FileExists(fileName) {
-			common.Exitf(1, "file %s already exists", fileName)
+			return fmt.Errorf("file %s already exists", fileName)
 		}
 		fmt.Printf("Downloading tarball list from %s\n", fileUrl)
-		err := rest.DownloadFile(fileName, fileUrl, true, globals.ProgressStepValue)
+		err := rest.DownloadFileWithRetry(fileName, fileUrl, true, globals.ProgressStepValue, retries)
 		if err != nil {
-			common.Exitf(1, "error downloading tarball list to JSON file")
+			return fmt.Errorf("error downloading tarball list to JSON file")
 		}
 	}
 	if !common.FileExists(fileName) {
-		common.Exitf(1, globals.ErrFileNotFound, fileName)
+		return fmt.Errorf(globals.ErrFileNotFound, fileName)
 	}
 	var tarballCollection downloads.TarballCollection
 
 	jsonText, err := common.SlurpAsBytes(fileName)
 	if err != nil {
-		common.Exitf(1, "error reading tarball list from %s", fileName)
+		return fmt.Errorf("error reading tarball list from %s", fileName)
 	}
 
 	err = json.Unmarshal(jsonText, &tarballCollection)
 	if err != nil {
-		common.Exitf(1, "error decoding tarball list from %s", fileName)
+		return fmt.Errorf("error decoding tarball list from %s", fileName)
 	}
 
 	err = downloads.TarballFileInfoValidation(tarballCollection)
 	if err != nil {
-		common.Exitf(1, "error validating tarball list from %s\n %s", fileName, err)
+		return fmt.Errorf("error validating tarball list from %s\n %s", fileName, err)
 	}
 	err = downloads.WriteTarballFileInfo(tarballCollection)
 	if err != nil {
-		common.Exitf(1, "error writing tarball list: %s", err)
+		return fmt.Errorf("error writing tarball list: %s", err)
 	}
 	fmt.Printf("Tarball list imported from %s to %s\n", fileName, downloads.TarballFileRegistry)
+	return nil
 }
 
 func addRemoteTarballToCollection(cmd *cobra.Command, args []string) {
@@ -652,7 +656,7 @@ its Github repository.
 (See: dbdeployer info defaults remote-tarball-url)
 `,
 
-	Run: importTarballCollection,
+	RunE: importTarballCollection,
 }
 
 var downloadsResetCmd = &cobra.Command{
@@ -717,6 +721,7 @@ func addCommonDownloadsFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolP(globals.QuietLabel, "", false, "Do not show download progress")
 	cmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
 	cmd.Flags().BoolP(globals.DeleteAfterUnpackLabel, "", false, "Delete the tarball after successful unpack")
+	cmd.Flags().Int64P(globals.RetriesOnFailure, "", 0, "How many times retry a download if a failure occurs on first try")
 }
 
 func init() {
@@ -749,25 +754,16 @@ func init() {
 	_ = downloadsTreeCmd.MarkFlagRequired(globals.FlavorLabel)
 
 	downloadsGetByVersionCmd.Flags().BoolP(globals.NewestLabel, "", false, "Choose only the newest tarballs not yet downloaded")
-	//downloadsGetByVersionCmd.Flags().BoolP(globals.DryRunLabel, "", false, "Show what would be downloaded, but don't run it")
 	downloadsGetByVersionCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Choose only minimal tarballs")
 	downloadsGetByVersionCmd.Flags().String(globals.FlavorLabel, "", "Choose only the given flavor")
 	downloadsGetByVersionCmd.Flags().String(globals.OSLabel, "", "Choose only the given OS")
-	//downloadsGetByVersionCmd.Flags().BoolP(globals.QuietLabel, "", false, "Do not show download progress")
 	downloadsGetByVersionCmd.Flags().BoolP(globals.GuessLatestLabel, "", false, "Guess the latest version (highest version w/ increased revision number)")
 	downloadsGetByVersionCmd.Flags().BoolP(globals.UnpackLabel, "", false, "Unpack after downloading")
-	//downloadsGetByVersionCmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
 	addCommonDownloadsFlags(downloadsGetByVersionCmd)
 
-	//downloadsGetCmd.Flags().BoolP(globals.QuietLabel, "", false, "Do not show download progress")
-	//downloadsGetCmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
-	//downloadsGetCmd.Flags().BoolP(globals.DryRunLabel, "", false, "Show what would be downloaded, but don't run it")
-	//downloadsGetCmd.Flags().String(globals.OSLabel, "", "Set the OS of the tarball")
 	downloadsGetCmd.Flags().BoolP(globals.UnpackLabel, "", false, "Unpack after downloading")
 	addCommonDownloadsFlags(downloadsGetCmd)
 
-	//downloadsGetUnpackCmd.Flags().BoolP(globals.DeleteAfterUnpackLabel, "", false, "Delete the tarball after successful unpack")
-	//downloadsGetUnpackCmd.Flags().Bool(globals.DryRunLabel, false, "Show Get operations, but do not run them")
 	downloadsGetUnpackCmd.PersistentFlags().String(globals.FlavorLabel, "", "Defines the tarball flavor (MySQL, NDB, Percona Server, etc)")
 	addCommonDownloadsFlags(downloadsGetUnpackCmd)
 
@@ -786,15 +782,7 @@ func init() {
 
 	downloadsAddStdinCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
 
-	// downloadsGetUnpack needs the same flags that cmdUnpack has
-	//downloadsGetUnpackCmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
-	//downloadsGetUnpackCmd.PersistentFlags().Int(globals.VerbosityLabel, 1, "Level of verbosity during unpack (0=none, 2=maximum)")
-	//downloadsGetUnpackCmd.PersistentFlags().String(globals.UnpackVersionLabel, "", "which version is contained in the tarball")
-	//downloadsGetUnpackCmd.PersistentFlags().String(globals.PrefixLabel, "", "Prefix for the final expanded directory")
-	//downloadsGetUnpackCmd.PersistentFlags().Bool(globals.ShellLabel, false, "Unpack a shell tarball into the corresponding server directory")
-	//downloadsGetUnpackCmd.PersistentFlags().Bool(globals.OverwriteLabel, false, "Overwrite the destination directory if already exists")
-	//downloadsGetUnpackCmd.PersistentFlags().String(globals.TargetServerLabel, "", "Uses a different server to unpack a shell tarball")
-	//downloadsGetUnpackCmd.PersistentFlags().String(globals.FlavorLabel, "", "Defines the tarball flavor (MySQL, NDB, Percona Server, etc)")
-
 	downloadsExportCmd.Flags().BoolP(globals.AddEmptyItemLabel, "", false, "Add an empty item to the tarballs list")
+
+	downloadsImportCmd.Flags().Int64P(globals.RetriesOnFailure, "", 0, "How many times retry a download if a failure occurs on first try")
 }