Merge branch 'develop' into fix-sync-master-to1.8

base/gfspapp/app_options.go

@@ -349,7 +349,7 @@ var bsdbOnce = sync.Once{}
 
 func DefaultGfBsDBOption(app *GfSpBaseApp, cfg *gfspconfig.GfSpConfig) error {
 	for _, v := range cfg.Server {
-		if v != coremodule.MetadataModularName {
+		if v != coremodule.MetadataModularName && v != coremodule.BlockSyncerModularName {
 			log.Infof("[%s] module doesn't need bs db", v)
 			continue
 		}

base/gfspclient/interface.go

@@ -156,6 +156,7 @@ type MetadataAPI interface {
 	SecondarySpIncomeDetails(ctx context.Context, spID uint32, opts ...grpc.DialOption) (int64, []*types.SecondarySpIncomeDetail, error)
 	GetBucketSize(ctx context.Context, bucketID uint64, opts ...grpc.DialOption) (string, error)
 	GetBucketInfoByBucketName(ctx context.Context, bucketName string, opts ...grpc.DialOption) (*types.Bucket, error)
+	GetBsDBInfo(ctx context.Context, blockHeight uint64, opts ...grpc.DialOption) (*types.GfSpGetBsDBInfoResponse, error)
 }
 
 // P2PAPI for mock use

base/gfspclient/metadata.go

@@ -1133,3 +1133,21 @@ func (s *GfSpClient) GetBucketReadQuotaCount(ctx context.Context, yearMonth stri
 	}
 	return resp.GetCount(), nil
 }
+
+func (s *GfSpClient) GetBsDBInfo(ctx context.Context, blockHeight uint64, opts ...grpc.DialOption) (*types.GfSpGetBsDBInfoResponse, error) {
+	conn, connErr := s.Connection(ctx, s.metadataEndpoint, opts...)
+	if connErr != nil {
+		log.CtxErrorw(ctx, "client failed to connect metadata", "error", connErr)
+		return nil, ErrRPCUnknownWithDetail("client failed to connect metadata, error: ", connErr)
+	}
+	defer conn.Close()
+	req := &types.GfSpGetBsDBInfoRequest{
+		BlockHeight: blockHeight,
+	}
+	resp, err := types.NewGfSpMetadataServiceClient(conn).GfSpGetBsDBInfo(ctx, req)
+	if err != nil {
+		log.CtxErrorw(ctx, "client failed to get bsDB data info", "error", err)
+		return nil, ErrRPCUnknownWithDetail("client failed to get get bsDB data info, error: ", err)
+	}
+	return resp, nil
+}

base/gfspconfig/config.go

@@ -267,10 +267,18 @@ type LogConfig struct {
 }
 
 type BlockSyncerConfig struct {
-	Modules          []string `comment:"required"`
-	Workers          uint     `comment:"required"`
-	CommitNumber     uint64   `comment:"optional"`
-	BsDBWriteAddress string   `comment:"optional"`
+	Modules                []string         `comment:"required"`
+	Workers                uint             `comment:"required"`
+	CommitNumber           uint64           `comment:"optional"`
+	BsDBWriteAddress       string           `comment:"optional"`
+	DataMonitor            bool             `comment:"optional"`
+	DataStatisticsDuration int64            `comment:"optinal"`
+	ChainDataStorage       ChainDataStorage `comment:"optional"`
+}
+
+type ChainDataStorage struct {
+	EnableStorage       bool   `comment:"optional"`
+	MaximumStorageCount uint64 `comment:"required"`
 }
 
 type MetadataConfig struct {

go.mod

@@ -308,7 +308,7 @@ replace (
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
 	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.6.1-0.20240419024340-b5c75cfd8110
 	github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1
-	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240326082854-3c67c32eb2be
+	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240422102216-0039530fcfb2
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 )

go.sum

@@ -192,8 +192,8 @@ github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965b
 github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM=
 github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM=
 github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI=
-github.com/bnb-chain/juno/v4 v4.0.0-20240326082854-3c67c32eb2be h1:yympDqs1gg3oUgSTm31ybKhvWgsEySvqC7LO1npVvRc=
-github.com/bnb-chain/juno/v4 v4.0.0-20240326082854-3c67c32eb2be/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
+github.com/bnb-chain/juno/v4 v4.0.0-20240422102216-0039530fcfb2 h1:Zpq40LEvqzaqtT7QqISASBg6E51SZPAWLQ5gdbAtsfk=
+github.com/bnb-chain/juno/v4 v4.0.0-20240422102216-0039530fcfb2/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=

modular/blocksyncer/blocksyncer.go

@@ -46,13 +46,17 @@ type MigrateDBKey struct{}
 
 // BlockSyncerModular synchronizes storage,payment,permission data to db by handling related events
 type BlockSyncerModular struct {
-	config    *config.TomlConfig
-	name      string
-	parserCtx *parser.Context
-	running   atomic.Value
-	context   context.Context
-	scope     rcmgr.ResourceScope
-	baseApp   *gfspapp.GfSpBaseApp
+	config                 *config.TomlConfig
+	name                   string
+	parserCtx              *parser.Context
+	running                atomic.Value
+	context                context.Context
+	scope                  rcmgr.ResourceScope
+	baseApp                *gfspapp.GfSpBaseApp
+	DataMonitorEnable      bool
+	DataStatisticsDuration int64
+	BlockResultStorage     bool
+	MaxBlockNum            int64
 }
 
 // Read concurrency required global variables

modular/blocksyncer/blocksyncer_indexer.go

@@ -27,15 +27,16 @@ import (
 	"github.com/bnb-chain/greenfield-storage-provider/pkg/metrics"
 )
 
-func NewIndexer(codec codec.Codec, proxy node.Node, db database.Database, modules []modules.Module, serviceName string, commitNumber uint64) parser.Indexer {
+func NewIndexer(codec codec.Codec, proxy node.Node, db database.Database, modules []modules.Module, serviceName string, commitNumber uint64, blockResultStorageEnable bool) parser.Indexer {
 	return &Impl{
-		codec:           codec,
-		Node:            proxy,
-		DB:              db,
-		Modules:         modules,
-		ServiceName:     serviceName,
-		ProcessedHeight: 0,
-		CommitNumber:    commitNumber,
+		codec:                    codec,
+		Node:                     proxy,
+		DB:                       db,
+		Modules:                  modules,
+		ServiceName:              serviceName,
+		ProcessedHeight:          0,
+		CommitNumber:             commitNumber,
+		BlockResultStorageEnable: blockResultStorageEnable,
 	}
 }
 
@@ -48,7 +49,8 @@ type Impl struct {
 	LatestBlockHeight atomic.Value
 	ProcessedHeight   uint64
 
-	CommitNumber uint64
+	CommitNumber             uint64
+	BlockResultStorageEnable bool
 
 	ServiceName string
 }
@@ -114,6 +116,9 @@ func (i *Impl) Process(height uint64) error {
 			return err
 		}
 		metrics.ChainRPCTime.Set(float64(time.Since(rpcStartTime).Milliseconds()))
+		if i.BlockResultStorageEnable {
+			go i.SaveBlockResult(height, events)
+		}
 		txHash = block.Block.Data.Txs
 		txs = make(map[common.Hash][]cometbfttypes.Event)
 		for idx := 0; idx < len(events.TxsResults); idx++ {
@@ -134,6 +139,9 @@ func (i *Impl) Process(height uint64) error {
 			log.Warnf("failed to get map data height: %d", height)
 			return ErrBlockNotFound
 		}
+		if i.BlockResultStorageEnable {
+			go i.SaveBlockResult(height, events)
+		}
 	}
 
 	startTime := time.Now()
@@ -412,3 +420,25 @@ func (i *Impl) CreateMasterTable() error {
 func (i *Impl) GetServiceName() string {
 	return i.ServiceName
 }
+
+func (i *Impl) SaveBlockResult(blockHeight uint64, result *coretypes.ResultBlockResults) {
+	res, err := json.Marshal(result)
+	if err != nil {
+		log.Errorw("failed to marshal block result", "error", err)
+		metrics.SaveBlockResultErr.Inc()
+		return
+	}
+	for n := 0; n < 5; n++ {
+		err = localDB.Cast(i.DB).SaveBlockResult(context.Background(), &models.BlockResult{
+			BlockHeight: blockHeight,
+			Result:      string(res),
+		})
+		if err == nil {
+			break
+		} else {
+			metrics.SaveBlockResultErr.Inc()
+			log.Errorw("failed to save block result", "error", err)
+		}
+	}
+
+}