Merge pull request #1396 from bnb-chain/fix-sync-master-to1.8

sync master master to develop

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v1.7.0
+BUGFIXES
+* [#1394](https://github.com/bnb-chain/greenfield-storage-provider/pull/1394)  fix: pick new gvg when retry failed replicate piece task
+* [#1391](https://github.com/bnb-chain/greenfield-storage-provider/pull/1391)  fix: check if it is AgentUploadTask
+* [#1390](https://github.com/bnb-chain/greenfield-storage-provider/pull/1390)  fix: delegate upload param check
+* [#1389](https://github.com/bnb-chain/greenfield-storage-provider/pull/1389)  fix: delegate upload param check
+* [#1387](https://github.com/bnb-chain/greenfield-storage-provider/pull/1387)  fix: upgrade deps for fixing vulnerabilities
+* [#1386](https://github.com/bnb-chain/greenfield-storage-provider/pull/1386)  fix: check if BucketExtraInfo is nil
+* [#1384](https://github.com/bnb-chain/greenfield-storage-provider/pull/1384)  fix: fix db override
+
+FEATURES
+* [#1392](https://github.com/bnb-chain/greenfield-storage-provider/pull/1392)  feat: provide recommended vgf
+
 ## v1.6.0
 
 BUGFIXES

core/task/null_task.go

@@ -204,3 +204,4 @@ func (t *NullTask) SetDelegateCreateObject(object *storagetypes.MsgDelegateCreat
 func (t *NullTask) GetIsAgentUpload() bool                                               { return false }
 func (t *NullTask) InitReplicatePieceTask(object *storagetypes.ObjectInfo, params *storagetypes.Params, priority TPriority, timeout int64, retry int64, isAgentUpload bool) {
 }
+func (t *NullTask) GetIsAgentUploadTask() bool { return false }

core/task/task.go

@@ -352,6 +352,8 @@ type ReceivePieceTask interface {
 	GetBucketMigration() bool
 	// SetBucketMigration sets the bucket migration
 	SetBucketMigration(bool)
+	// GetIsAgentUploadTask set the is agent upload flag
+	GetIsAgentUploadTask() bool
 }
 
 // SealObjectTask is an abstract interface to record the information for sealing object on Greenfield chain.

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/bnb-chain/greenfield v1.6.0
 	github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a
 	github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6
-	github.com/cometbft/cometbft v0.38.0
+	github.com/cometbft/cometbft v0.38.6
 	github.com/consensys/gnark-crypto v0.7.0
 	github.com/cosmos/cosmos-proto v1.0.0-beta.3
 	github.com/cosmos/cosmos-sdk v0.47.10
@@ -44,7 +44,7 @@ require (
 	go.uber.org/mock v0.2.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
 	golang.org/x/time v0.3.0
 	google.golang.org/grpc v1.59.0
@@ -277,7 +277,7 @@ require (
 	go.uber.org/fx v1.19.2 // indirect
 	golang.org/x/arch v0.5.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
@@ -303,10 +303,10 @@ replace (
 	cosmossdk.io/api => github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231206043955-0855e0965bc8
 	cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8
 	github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.23.0
-	github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v0.0.0-20240402065323-40677309d454
+	github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v1.2.1-0.20240408033601-a6b682aa870e
 	github.com/cometbft/cometbft-db => github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
-	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.6.0
+	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.6.1-0.20240419024340-b5c75cfd8110
 	github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1
 	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240422102216-0039530fcfb2
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1

go.sum

@@ -178,14 +178,14 @@ github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/bnb-chain/greenfield v1.6.0 h1:AAEJjtwED6XVckPRqMSa6nUciEeBNszeJhNiO/2KNiE=
 github.com/bnb-chain/greenfield v1.6.0/go.mod h1:KCjy1bsc5q2G4IjyhKX0r3BQM+4rooZ1/fNnGlq8o3I=
-github.com/bnb-chain/greenfield-cometbft v0.0.0-20240402065323-40677309d454 h1:mLXDQRfTUbOKOrbaaxbTsVy6V2kZhPZmsIPwsyIGBTs=
-github.com/bnb-chain/greenfield-cometbft v0.0.0-20240402065323-40677309d454/go.mod h1:q9/nqW19iXvxyma5XgcZfxL/OkWI9s5e7yX9ecePz8A=
+github.com/bnb-chain/greenfield-cometbft v1.2.1-0.20240408033601-a6b682aa870e h1:4ttDy8yBhBUW0gdFyBK0wHMJS5ZtlhBdoYx/O6T6Eqg=
+github.com/bnb-chain/greenfield-cometbft v1.2.1-0.20240408033601-a6b682aa870e/go.mod h1:q9/nqW19iXvxyma5XgcZfxL/OkWI9s5e7yX9ecePz8A=
 github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 h1:XcWulGacHVRiSCx90Q8Y//ajOrLNBQWR/KDB89dy3cU=
 github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1/go.mod h1:ey1CiK4bYo1RBNJLRiVbYr5CMdSxci9S/AZRINLtppI=
 github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a h1:VjUknQkIcqkjYCt1hmfpinM7kToOBuUU+KykrrqFsEM=
 github.com/bnb-chain/greenfield-common/go v0.0.0-20240228080631-2683b0ee669a/go.mod h1:K9jK80fbahciC+FAvrch8Qsbw9ZkvVgjfKsqrzPTAVA=
-github.com/bnb-chain/greenfield-cosmos-sdk v1.6.0 h1:La26hQZ+as0vrgbR5ppjz8rDFH4TgYLJd8ctG7z9zNQ=
-github.com/bnb-chain/greenfield-cosmos-sdk v1.6.0/go.mod h1:XF8U3VN1euzLkIR5xiSNyQSnBabvnD86oz6fgdrpteQ=
+github.com/bnb-chain/greenfield-cosmos-sdk v1.6.1-0.20240419024340-b5c75cfd8110 h1:max1dH2HkKrNZpL2Jv6xwl+XWHsjJC6Ay+caN17u3CI=
+github.com/bnb-chain/greenfield-cosmos-sdk v1.6.1-0.20240419024340-b5c75cfd8110/go.mod h1:siglWrVkM1+6tj8ZPwzMIITWQh7D8gsKJUk0Suz+ul0=
 github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231206043955-0855e0965bc8 h1:mUMOeNo3K0SZvAhiOHNKW4mmkrhOphBF8tDUyK6e1tY=
 github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231206043955-0855e0965bc8/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM=
 github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8 h1:1Ud7itq03c4Q9h0kBpw1FYlWKN3kco8cgj59vdd50UQ=
@@ -1742,8 +1742,8 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1859,8 +1859,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

modular/authenticator/authenticator.go

@@ -450,8 +450,10 @@ func (a *AuthenticationModular) VerifyAuthentication(
 			}
 			return false, ErrConsensusWithDetail("failed to get bucket extra info from consensus, error: " + err.Error())
 		}
-		if bucketExtraInfo.IsRateLimited {
-			return false, ErrBucketIsRateLimited
+		if bucketExtraInfo != nil {
+			if bucketExtraInfo.IsRateLimited {
+				return false, ErrBucketIsRateLimited
+			}
 		}
 		spID, err := a.getSPID()
 		if err != nil {

modular/executor/execute_replicate.go

@@ -317,7 +317,6 @@ func (e *ExecuteModular) doReplicatePiece(ctx context.Context, waitGroup *sync.W
 	}
 	receive.SetSignature(signature)
 	replicateOnePieceTime := time.Now()
-
 	if err = retry.Do(func() error {
 		// timeout for single piece replication
 		ctxWithTimeout, cancel := context.WithTimeout(ctx, replicateTimeOut)

modular/gater/admin_handler.go

@@ -641,12 +641,20 @@ func (g *GateModular) replicateHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if receiveTask.GetObjectInfo() == nil || (!receiveTask.GetIsAgentUploadTask() && int(receiveTask.GetRedundancyIdx()) >= len(receiveTask.GetObjectInfo().GetChecksums())) {
+	log.CtxInfow(reqCtx.Context(), "debugInfo", "receiveTask", receiveTask)
+
+	if receiveTask.GetObjectInfo() == nil {
 		log.CtxErrorw(reqCtx.Context(), "receive task params error")
 		err = ErrInvalidHeader
 		return
 	}
 
+	if !receiveTask.GetIsAgentUploadTask() && int(receiveTask.GetRedundancyIdx()) >= len(receiveTask.GetObjectInfo().GetChecksums()) {
+		log.CtxErrorw(reqCtx.Context(), "receive task params error", "object_id", receiveTask.GetObjectInfo().Id, "object_name", receiveTask.GetObjectInfo().GetObjectName())
+		err = ErrInvalidHeader
+		return
+	}
+
 	readDataTime := time.Now()
 	data, err = io.ReadAll(r.Body)
 	metrics.PerfReceivePieceTimeHistogram.WithLabelValues("receive_piece_read_piece_time").Observe(time.Since(readDataTime).Seconds())

modular/gater/bucket_handler.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/bnb-chain/greenfield-storage-provider/base/types/gfsperrors"
 	"github.com/bnb-chain/greenfield-storage-provider/base/types/gfspserver"
+	"github.com/bnb-chain/greenfield-storage-provider/base/types/gfsptask"
 	coremodule "github.com/bnb-chain/greenfield-storage-provider/core/module"
 	modelgateway "github.com/bnb-chain/greenfield-storage-provider/model/gateway"
 	metadatatypes "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types"
@@ -444,3 +445,63 @@ func (g *GateModular) getBucketReadQuotaCountHandler(w http.ResponseWriter, r *h
 	}
 	log.CtxDebugw(ctx, "succeed to get bucket quota count", "xml_info", xmlInfo)
 }
+
+func (g *GateModular) getRecommendedVGFIDHandler(w http.ResponseWriter, r *http.Request) {
+	var (
+		err    error
+		reqCtx *RequestContext
+	)
+	startTime := time.Now()
+	defer func() {
+		reqCtx.Cancel()
+		if err != nil {
+			reqCtx.SetError(gfsperrors.MakeGfSpError(err))
+			reqCtx.SetHTTPCode(int(gfsperrors.MakeGfSpError(err).GetHttpStatusCode()))
+			modelgateway.MakeErrorResponse(w, gfsperrors.MakeGfSpError(err))
+			metrics.ReqCounter.WithLabelValues(GatewayTotalFailure).Inc()
+			metrics.ReqTime.WithLabelValues(GatewayTotalFailure).Observe(time.Since(startTime).Seconds())
+		} else {
+			reqCtx.SetHTTPCode(http.StatusOK)
+			metrics.ReqCounter.WithLabelValues(GatewayTotalSuccess).Inc()
+			metrics.ReqTime.WithLabelValues(GatewayTotalSuccess).Observe(time.Since(startTime).Seconds())
+		}
+		log.CtxDebugw(reqCtx.Context(), reqCtx.String())
+	}()
+
+	reqCtx, err = NewRequestContext(r, g)
+	if err != nil {
+		return
+	}
+
+	vgfID, err := g.baseApp.GfSpClient().PickVirtualGroupFamilyID(
+		reqCtx.Context(), &gfsptask.GfSpCreateBucketApprovalTask{
+			Task:             &gfsptask.GfSpTask{},
+			CreateBucketInfo: &storagetypes.MsgCreateBucket{},
+		})
+	if err != nil {
+		log.CtxErrorw(reqCtx.Context(), "failed to get recommended virtual group family", "error", err)
+		return
+	}
+
+	var xmlInfo = struct {
+		XMLName xml.Name `xml:"VirtualGroupFamily"`
+		Id      uint32   `xml:"Id"`
+	}{
+		Id: vgfID,
+	}
+
+	xmlBody, err := xml.Marshal(&xmlInfo)
+	if err != nil {
+		log.Errorw("failed to marshal xml", "error", err)
+		err = ErrEncodeResponseWithDetail("failed to marshal xml, error: " + err.Error())
+		return
+	}
+	w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue)
+
+	if _, err = w.Write(xmlBody); err != nil {
+		log.Errorw("failed to write body", "error", err)
+		err = ErrEncodeResponseWithDetail("failed to write body, error: " + err.Error())
+		return
+	}
+	log.CtxDebugw(reqCtx.Context(), "succeed to get recommended virtual group family", "xml_info", xmlInfo)
+}

modular/gater/const.go

@@ -34,6 +34,8 @@ const (
 
 	// GetApprovalPath defines get-approval path style suffix
 	GetApprovalPath = "/greenfield/admin/v1/get-approval"
+	// GetRecommendedVirtualGroupFamilyPath defines get-recommended-vgf path style suffix
+	GetRecommendedVirtualGroupFamilyPath = "/greenfield/admin/v1/get-recommended-vgf"
 	// ActionQuery defines get-approval's type, currently include create bucket and create object
 	ActionQuery = "action"
 	// UploadProgressQuery defines upload progress query, which is used to route request

modular/gater/object_handler.go

@@ -1085,7 +1085,7 @@ func (g *GateModular) delegatePutObjectHandler(w http.ResponseWriter, r *http.Re
 		err = ErrNoPermission
 		return
 	}
-	contentType = reqCtx.vars["content_type"]
+	contentType = r.Header.Get(ContentTypeHeader)
 	if contentType == "" {
 		contentType = ContentDefault
 	}
@@ -1154,10 +1154,12 @@ func (g *GateModular) delegatePutObjectHandler(w http.ResponseWriter, r *http.Re
 		if err != nil && !strings.Contains(err.Error(), "No such object") {
 			log.CtxErrorw(reqCtx.ctx, "failed to QueryObjectInfo", "error", err)
 			return
-		} else if objectInfo != nil && (objectInfo.ObjectStatus != storagetypes.OBJECT_STATUS_CREATED || objectInfo.Creator != reqCtx.account || objectInfo.PayloadSize != payloadSize) {
+		}
+		if objectInfo != nil && (objectInfo.ObjectStatus != storagetypes.OBJECT_STATUS_CREATED || (objectInfo.Creator != reqCtx.account && objectInfo.Owner != reqCtx.account) || objectInfo.PayloadSize != payloadSize) {
 			err = ErrInvalidQuery
 			return
-		} else {
+		}
+		if objectInfo == nil {
 			var visibilityInt int64
 			visibilityStr := queryParams.Get("visibility")
 			visibilityInt, err = strconv.ParseInt(visibilityStr, 10, 32)
@@ -1353,6 +1355,10 @@ func (g *GateModular) delegateResumablePutObjectHandler(w http.ResponseWriter, r
 		return
 	}
 	fingerprint = commonhash.GenerateChecksum(approvalMsg)
+	contentType = r.Header.Get(ContentTypeHeader)
+	if contentType == "" {
+		contentType = ContentDefault
+	}
 	startTime := time.Now()
 
 	if isUpdate {
@@ -1571,7 +1577,7 @@ func (g *GateModular) delegateCreateFolderHandler(w http.ResponseWriter, r *http
 		err = ErrNoPermission
 		return
 	}
-	contentType = reqCtx.vars["content_type"]
+	contentType = r.Header.Get(ContentTypeHeader)
 	if contentType == "" {
 		contentType = ContentDefault
 	}

modular/gater/router.go

@@ -86,6 +86,7 @@ const (
 	getSPMigratingBucketNumberRouterName           = "GetSPMigratingBucketNumber"
 	verifyMigrateGVGPermissionRouterName           = "VerifyMigrateGVGPermission"
 	getBucketSizeRouterName                        = "GetBucketSize"
+	getRecommendedVGFRouterName                    = "GetRecommendedVGF"
 	getBsDBDataInfo                                = "GetBsDBDataInfo"
 )
 
@@ -121,6 +122,9 @@ func (g *GateModular) RegisterHandler(router *mux.Router) {
 	router.Path(GetApprovalPath).Name(approvalRouterName).Methods(http.MethodGet).HandlerFunc(g.getApprovalHandler).
 		Queries(ActionQuery, "{action}")
 
+	// Query recommended virtual group family for creating bucket
+	router.Path(GetRecommendedVirtualGroupFamilyPath).Name(getRecommendedVGFRouterName).Methods(http.MethodGet).HandlerFunc(g.getRecommendedVGFIDHandler)
+
 	// get challenge info
 	router.Path(GetChallengeInfoPath).Name(getChallengeInfoRouterName).Methods(http.MethodGet).HandlerFunc(g.getChallengeInfoHandler)
 	router.Path(GetChallengeInfoV2Path).Name(getChallengeInfoV2RouterName).Methods(http.MethodGet).HandlerFunc(g.getChallengeInfoV2Handler)

modular/manager/manager.go

@@ -508,23 +508,25 @@ func (m *ManageModular) LoadTaskFromDB() error {
 		replicateTask.InitReplicatePieceTask(objectInfo, storageParams, m.baseApp.TaskPriority(replicateTask),
 			m.baseApp.TaskTimeout(replicateTask, objectInfo.GetPayloadSize()), m.baseApp.TaskMaxRetry(replicateTask), meta.IsAgentUpload)
 
-		if meta.GlobalVirtualGroupID == 0 {
-			bucketInfo, err := m.baseApp.GfSpClient().GetBucketByBucketName(context.Background(), objectInfo.BucketName, true)
-			if err != nil || bucketInfo == nil {
-				log.Errorw("failed to get bucket by bucket name", "bucket", bucketInfo, "error", err)
-				return err
-			}
-			gvgMeta, err := m.pickGlobalVirtualGroup(context.Background(), bucketInfo.BucketInfo.GlobalVirtualGroupFamilyId, storageParams)
-			log.Infow("pick global virtual group", "gvg_meta", gvgMeta, "error", err)
-			if err != nil {
-				return err
-			}
-			replicateTask.GlobalVirtualGroupId = gvgMeta.ID
-			replicateTask.SecondaryEndpoints = gvgMeta.SecondarySPEndpoints
-		} else {
-			replicateTask.GlobalVirtualGroupId = meta.GlobalVirtualGroupID
-			replicateTask.SecondaryEndpoints = meta.SecondaryEndpoints
+		//retrieve objects from the database that have not completed the replicate piece, reselect gvg, and then add them to the replicate queue
+		bucketInfo, err := m.baseApp.GfSpClient().GetBucketByBucketName(context.Background(), objectInfo.BucketName, true)
+		if err != nil || bucketInfo == nil {
+			log.Errorw("failed to get bucket by bucket name", "bucket", bucketInfo, "error", err)
+			return err
+		}
+		gvgMeta, err := m.pickGlobalVirtualGroup(context.Background(), bucketInfo.BucketInfo.GlobalVirtualGroupFamilyId, storageParams)
+		log.Infow("pick global virtual group", "gvg_meta", gvgMeta, "error", err)
+		if err != nil {
+			return err
+		}
+		replicateTask.GlobalVirtualGroupId = gvgMeta.ID
+		replicateTask.SecondaryEndpoints = gvgMeta.SecondarySPEndpoints
+		meta.GlobalVirtualGroupID = gvgMeta.ID
+		meta.SecondaryEndpoints = gvgMeta.SecondarySPEndpoints
+		if err = m.baseApp.GfSpDB().UpdateUploadProgress(meta); err != nil {
+			log.Errorw("failed to update object task state", "task_info", replicateTask.Info(), "error", err)
 		}
+
 		pushErr := m.replicateQueue.Push(replicateTask)
 		if pushErr != nil {
 			log.Errorw("failed to push replicate piece task to queue", "object_info", objectInfo, "error", pushErr)

modular/manager/manager_test.go

@@ -239,7 +239,10 @@ func TestManageModular_LoadTaskFromDB(t *testing.T) {
 	m3.EXPECT().GetBucketByBucketName(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&types.Bucket{BucketInfo: &types0.BucketInfo{
 			GlobalVirtualGroupFamilyId: 1,
-		}}, nil)
+		}}, nil).AnyTimes()
+
+	m1.EXPECT().UpdateUploadProgress(gomock.Any()).Return(
+		nil).AnyTimes()
 
 	vgm := vgmgr.NewMockVirtualGroupManager(ctrl)
 	manage.virtualGroupManager = vgm