feat: improve upload and fix ctrl c

pkg/cmd/record/create.go

@@ -22,10 +22,7 @@ import (
 	openv1alpha1resource "buf.build/gen/go/coscene-io/coscene-openapi/protocolbuffers/go/coscene/openapi/dataplatform/v1alpha1/resources"
 	"github.com/coscene-io/cocli/internal/config"
 	"github.com/coscene-io/cocli/internal/name"
-	"github.com/coscene-io/cocli/pkg/cmd_utils"
 	"github.com/coscene-io/cocli/pkg/cmd_utils/upload_utils"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -86,26 +83,12 @@ func NewCreateCommand(cfgPath *string) *cobra.Command {
 				}
 
 				fmt.Println("Uploading thumbnail to pre-signed url...")
-				generateSecurityTokenRes, err := pm.SecurityTokenCli().GenerateSecurityToken(context.Background(), proj.String())
+				um, err := upload_utils.NewUploadManagerFromConfig(pm, proj, timeout, multiOpts)
 				if err != nil {
-					log.Fatalf("unable to generate security token: %v", err)
+					log.Fatalf("unable to create upload manager: %v", err)
 				}
 
-				mc, err := minio.New(pm.GetCurrentProfile().EndPoint, &minio.Options{
-					Creds:     credentials.NewStaticV4(generateSecurityTokenRes.GetAccessKeyId(), generateSecurityTokenRes.GetAccessKeySecret(), generateSecurityTokenRes.GetSessionToken()),
-					Secure:    true,
-					Transport: cmd_utils.NewTransport(timeout),
-				})
-				if err != nil {
-					log.Fatalf("unable to create minio client: %v", err)
-				}
-
-				um, err := upload_utils.NewUploadManager(mc, multiOpts)
-				if err != nil {
-					log.Fatalf("Failed to create upload manager: %v", err)
-				}
-
-				err = cmd_utils.UploadFileThroughUrl(um, thumbnail, thumbnailUploadUrl)
+				err = um.UploadFileThroughUrl(thumbnail, thumbnailUploadUrl)
 				if err != nil {
 					log.Fatalf("Failed to upload thumbnail: %v", err)
 				}

pkg/cmd/record/upload.go

@@ -27,10 +27,7 @@ import (
 	"github.com/coscene-io/cocli/internal/config"
 	"github.com/coscene-io/cocli/internal/fs"
 	"github.com/coscene-io/cocli/internal/name"
-	"github.com/coscene-io/cocli/pkg/cmd_utils"
 	"github.com/coscene-io/cocli/pkg/cmd_utils/upload_utils"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -85,20 +82,7 @@ func NewUploadCommand(cfgPath *string) *cobra.Command {
 			fmt.Printf("Uploading files to record: %s\n", recordName.RecordID)
 
 			// create minio client and upload manager first.
-			generateSecurityTokenRes, err := pm.SecurityTokenCli().GenerateSecurityToken(context.Background(), proj.String())
-			if err != nil {
-				log.Fatalf("unable to generate security token: %v", err)
-			}
-			mc, err := minio.New(generateSecurityTokenRes.Endpoint, &minio.Options{
-				Creds:     credentials.NewStaticV4(generateSecurityTokenRes.GetAccessKeyId(), generateSecurityTokenRes.GetAccessKeySecret(), generateSecurityTokenRes.GetSessionToken()),
-				Secure:    true,
-				Region:    "",
-				Transport: cmd_utils.NewTransport(timeout),
-			})
-			if err != nil {
-				log.Fatalf("unable to create minio client: %v", err)
-			}
-			um, err := upload_utils.NewUploadManager(mc, multiOpts)
+			um, err := upload_utils.NewUploadManagerFromConfig(pm, proj, timeout, multiOpts)
 			if err != nil {
 				log.Fatalf("unable to create upload manager: %v", err)
 			}
@@ -117,21 +101,14 @@ func NewUploadCommand(cfgPath *string) *cobra.Command {
 
 					fileAbsolutePath := path.Join(relativeDir, fileResource.Filename)
 
-					if err = cmd_utils.UploadFileThroughUrl(um, fileAbsolutePath, uploadUrl); err != nil {
+					if err = um.UploadFileThroughUrl(fileAbsolutePath, uploadUrl); err != nil {
 						um.AddErr(fileAbsolutePath, errors.Wrapf(err, "unable to upload file"))
 						continue
 					}
 				}
 			}
 
 			um.Wait()
-			if len(um.Errs) > 0 {
-				fmt.Printf("\n%d files failed to upload\n", len(um.Errs))
-				for kPath, vErr := range um.Errs {
-					fmt.Printf("Upload %v failed with: \n%v\n\n", kPath, vErr)
-				}
-				return
-			}
 
 			recordUrl, err := pm.GetRecordUrl(recordName)
 			if err == nil {

pkg/cmd_utils/upload_utils/heap.go

@@ -48,3 +48,22 @@ func (h *IntHeap) Remove(x any) {
 	}
 	heap.Init(h)
 }
+
+// FindMinMissingInteger finds the minimum missing integer in a sorted array
+func FindMinMissingInteger(arr []int) int {
+	left, right := 0, len(arr)
+
+	for left < right {
+		mid := (left + right) / 2
+
+		// Check if arr[mid] is equal to the expected value mid + 1
+		if arr[mid] == mid+1 {
+			left = mid + 1
+		} else {
+			right = mid
+		}
+	}
+
+	// The first missing number is left + 1
+	return left + 1
+}