Merge pull request #1382 from bnb-chain/fix-sync-master-to1.7

sync master master to develop

CHANGELOG.md

@@ -1,4 +1,17 @@
 # Changelog
+
+## v1.6.0
+
+BUGFIXES
+* [#1375](https://github.com/bnb-chain/greenfield-storage-provider/pull/1375)  fix: fix GC issue
+* [#1379](https://github.com/bnb-chain/greenfield-storage-provider/pull/1379)  perf: replicate piece
+* [#1378](https://github.com/bnb-chain/greenfield-storage-provider/pull/1378)  fix: deletion on checksum not fully performed and fail to override duplicate checksum
+
+FEATURES
+* [#1354](https://github.com/bnb-chain/greenfield-storage-provider/pull/1354)  feat: simplify off-chain-auth
+* [#1353](https://github.com/bnb-chain/greenfield-storage-provider/pull/1353)  feat: Primary SP as the upload agent
+* [#1351](https://github.com/bnb-chain/greenfield-storage-provider/pull/1351)  feat: sp monthly free quota
+
 ## v1.5.0
 
 BUGFIXES

base/gfspclient/interface.go

@@ -116,7 +116,9 @@ type MetadataAPI interface {
 	VerifyPermission(ctx context.Context, Operator string, bucketName string, objectName string, actionType permission_types.ActionType, opts ...grpc.DialOption) (*permission_types.Effect, error)
 	GetBucketMeta(ctx context.Context, bucketName string, includePrivate bool, opts ...grpc.DialOption) (*types.VGFInfoBucket, *payment_types.StreamRecord, error)
 	GetEndpointBySpID(ctx context.Context, spID uint32, opts ...grpc.DialOption) (string, error)
-	GetBucketReadQuota(ctx context.Context, bucket *storagetypes.BucketInfo, yearMonth string, opts ...grpc.DialOption) (uint64, uint64, uint64, uint64, error)
+	GetBucketReadQuota(ctx context.Context, bucket *storagetypes.BucketInfo, yearMonth string, opts ...grpc.DialOption) (uint64, uint64, uint64, uint64, uint64, uint64, error)
+	ListBucketReadQuota(ctx context.Context, yearMonth string, offset, limit uint32, opts ...grpc.DialOption) ([]*types.BucketReadQuotaRecord, error)
+	GetBucketReadQuotaCount(ctx context.Context, yearMonth string, opts ...grpc.DialOption) (int64, error)
 	GetLatestBucketReadQuota(ctx context.Context, bucketID uint64, opts ...grpc.DialOption) (*gfsptask.GfSpBucketQuotaInfo, error)
 	ListBucketReadRecord(ctx context.Context, bucket *storagetypes.BucketInfo, startTimestampUs, endTimestampUs, maxRecordNum int64, opts ...grpc.DialOption) ([]*types.ReadRecord, int64, error)
 	GetUploadObjectState(ctx context.Context, objectID uint64, opts ...grpc.DialOption) (int32, string, error)

base/gfspclient/metadata.go

@@ -332,11 +332,11 @@ func (s *GfSpClient) GetEndpointBySpID(ctx context.Context, spId uint32, opts ..
 }
 
 func (s *GfSpClient) GetBucketReadQuota(ctx context.Context, bucket *storage_types.BucketInfo, yearMonth string, opts ...grpc.DialOption) (
-	uint64, uint64, uint64, uint64, error) {
+	uint64, uint64, uint64, uint64, uint64, uint64, error) {
 	conn, connErr := s.Connection(ctx, s.metadataEndpoint, opts...)
 	if connErr != nil {
 		log.CtxErrorw(ctx, "client failed to connect metadata", "error", connErr)
-		return uint64(0), uint64(0), uint64(0), uint64(0), ErrRPCUnknownWithDetail("client failed to connect metadata, error: ", connErr)
+		return uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), ErrRPCUnknownWithDetail("client failed to connect metadata, error: ", connErr)
 	}
 	defer conn.Close()
 	req := &types.GfSpGetBucketReadQuotaRequest{
@@ -346,12 +346,35 @@ func (s *GfSpClient) GetBucketReadQuota(ctx context.Context, bucket *storage_typ
 	resp, err := types.NewGfSpMetadataServiceClient(conn).GfSpGetBucketReadQuota(ctx, req)
 	if err != nil {
 		log.CtxErrorw(ctx, "client failed to get bucket read quota", "error", err)
-		return uint64(0), uint64(0), uint64(0), uint64(0), ErrRPCUnknownWithDetail("client failed to get bucket read quota, error: ", err)
+		return uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), ErrRPCUnknownWithDetail("client failed to get bucket read quota, error: ", err)
 	}
 	if resp.GetErr() != nil {
-		return uint64(0), uint64(0), uint64(0), uint64(0), resp.GetErr()
+		return uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), resp.GetErr()
 	}
-	return resp.GetChargedQuotaSize(), resp.GetSpFreeQuotaSize(), resp.GetConsumedSize(), resp.FreeQuotaConsumeSize, nil
+	return resp.GetChargedQuotaSize(), resp.GetSpFreeQuotaSize(), resp.GetConsumedSize(), resp.FreeQuotaConsumeSize, resp.GetSpMonthlyFreeQuotaSize(), resp.GetMonthlyFreeQuotaConsumeSize(), nil
+}
+
+func (s *GfSpClient) ListBucketReadQuota(ctx context.Context, yearMonth string, offset, limit uint32, opts ...grpc.DialOption) ([]*types.BucketReadQuotaRecord, error) {
+	conn, connErr := s.Connection(ctx, s.metadataEndpoint, opts...)
+	if connErr != nil {
+		log.CtxErrorw(ctx, "client failed to connect metadata", "error", connErr)
+		return nil, ErrRPCUnknownWithDetail("client failed to connect metadata, error: ", connErr)
+	}
+	defer conn.Close()
+	req := &types.GfSpListBucketReadQuotaRequest{
+		YearMonth: yearMonth,
+		Offset:    offset,
+		Limit:     limit,
+	}
+	resp, err := types.NewGfSpMetadataServiceClient(conn).GfSpListBucketReadQuota(ctx, req)
+	if err != nil {
+		log.CtxErrorw(ctx, "client failed to list bucket read quota", "error", err)
+		return nil, ErrRPCUnknownWithDetail("client failed to list bucket read quota, error: ", err)
+	}
+	if resp.GetErr() != nil {
+		return nil, resp.GetErr()
+	}
+	return resp.GetResult(), nil
 }
 
 func (s *GfSpClient) GetLatestBucketReadQuota(ctx context.Context, bucketID uint64, opts ...grpc.DialOption) (
@@ -1089,3 +1112,24 @@ func (s *GfSpClient) GetBucketInfoByBucketName(ctx context.Context, bucketName s
 	}
 	return resp.GetBucket(), nil
 }
+
+func (s *GfSpClient) GetBucketReadQuotaCount(ctx context.Context, yearMonth string, opts ...grpc.DialOption) (int64, error) {
+	conn, connErr := s.Connection(ctx, s.metadataEndpoint, opts...)
+	if connErr != nil {
+		log.CtxErrorw(ctx, "client failed to connect metadata", "error", connErr)
+		return 0, ErrRPCUnknownWithDetail("client failed to connect metadata, error: ", connErr)
+	}
+	defer conn.Close()
+	req := &types.GfSpGetBucketReadQuotaCountRequest{
+		YearMonth: yearMonth,
+	}
+	resp, err := types.NewGfSpMetadataServiceClient(conn).GfSpGetBucketReadQuotaCount(ctx, req)
+	if err != nil {
+		log.CtxErrorw(ctx, "client failed to get bucket read quota count", "error", err)
+		return 0, ErrRPCUnknownWithDetail("client failed to get bucket read quota count, error: ", err)
+	}
+	if resp.GetErr() != nil {
+		return 0, resp.GetErr()
+	}
+	return resp.GetCount(), nil
+}

base/gnfd/gnfd_service.go

@@ -509,6 +509,36 @@ func (g *Gnfd) QueryBucketInfo(ctx context.Context, bucket string) (bucketInfo *
 	return resp.GetBucketInfo(), nil
 }
 
+// QueryBucketExtraInfo returns the bucket extra info by name.
+func (g *Gnfd) QueryBucketExtraInfo(ctx context.Context, bucket string) (bucketInfo *storagetypes.BucketExtraInfo, err error) {
+	startTime := time.Now()
+	defer func() {
+		if err != nil {
+			metrics.GnfdChainCounter.WithLabelValues(ChainFailureQueryBucketInfo).Inc()
+			metrics.GnfdChainTime.WithLabelValues(ChainFailureQueryBucketInfo).Observe(
+				time.Since(startTime).Seconds())
+			metrics.GnfdChainCounter.WithLabelValues(ChainFailureTotal).Inc()
+			metrics.GnfdChainTime.WithLabelValues(ChainFailureTotal).Observe(
+				time.Since(startTime).Seconds())
+			return
+		}
+		metrics.GnfdChainCounter.WithLabelValues(ChainSuccessQueryBucketInfo).Inc()
+		metrics.GnfdChainTime.WithLabelValues(ChainSuccessQueryBucketInfo).Observe(
+			time.Since(startTime).Seconds())
+		metrics.GnfdChainCounter.WithLabelValues(ChainSuccessTotal).Inc()
+		metrics.GnfdChainTime.WithLabelValues(ChainSuccessTotal).Observe(
+			time.Since(startTime).Seconds())
+	}()
+
+	client := g.getCurrentClient().GnfdClient()
+	resp, err := client.HeadBucket(ctx, &storagetypes.QueryHeadBucketRequest{BucketName: bucket})
+	if err != nil {
+		log.CtxErrorw(ctx, "failed to query bucket", "bucket_name", bucket, "error", err)
+		return nil, err
+	}
+	return resp.GetExtraInfo(), nil
+}
+
 // QueryBucketInfoById returns the bucket info by name.
 func (g *Gnfd) QueryBucketInfoById(ctx context.Context, bucketId uint64) (bucketInfo *storagetypes.BucketInfo, err error) {
 	startTime := time.Now()

base/types/gfsptask/gc.go

@@ -313,12 +313,15 @@ func (m *GfSpGCStaleVersionObjectTask) InitGCStaleVersionObjectTask(priority cor
 	redundancyIndex int32,
 	integrityChecksum []byte,
 	pieceChecksumList [][]byte,
-	version, timeout int64) {
+	version int64,
+	objectSize uint64,
+	timeout int64) {
 	m.Reset()
 	m.Task = &GfSpTask{}
 	m.SetObjectID(objectID)
 	m.SetRedundancyIndex(redundancyIndex)
 	m.SetVersion(version)
+	m.SetObjectSize(objectSize)
 	m.SetIntegrityChecksum(integrityChecksum)
 	m.SetPieceChecksumList(pieceChecksumList)
 	m.SetPriority(priority)
@@ -454,6 +457,10 @@ func (m *GfSpGCStaleVersionObjectTask) SetVersion(version int64) {
 	m.Version = version
 }
 
+func (m *GfSpGCStaleVersionObjectTask) SetObjectSize(objectSize uint64) {
+	m.ObjectSize = objectSize
+}
+
 func (m *GfSpGCStaleVersionObjectTask) SetRedundancyIndex(redundancyIndex int32) {
 	m.RedundancyIndex = redundancyIndex
 }