fix: data statistics by read-only db

go.mod

@@ -308,7 +308,7 @@ replace (
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
 	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.6.0
 	github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1
-	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240410092656-8d655b8c3f2d
+	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240415080646-269767e40736
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 )

go.sum

@@ -192,8 +192,8 @@ github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965b
 github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM=
 github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM=
 github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI=
-github.com/bnb-chain/juno/v4 v4.0.0-20240410092656-8d655b8c3f2d h1:MJNPGV+pWPqSPk0zWV67wU5Ev5wez/glHhEV0kQklJo=
-github.com/bnb-chain/juno/v4 v4.0.0-20240410092656-8d655b8c3f2d/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
+github.com/bnb-chain/juno/v4 v4.0.0-20240415080646-269767e40736 h1:B464Z533SEPBfv3mjocuuajFa/Fb3S80ZucUTrTDZd0=
+github.com/bnb-chain/juno/v4 v4.0.0-20240415080646-269767e40736/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=

modular/blocksyncer/blocksyncer_options.go

@@ -403,7 +403,7 @@ func (b *BlockSyncerModular) dbStatistics(ctx context.Context) {
 	}
 	ticker := time.NewTicker(time.Duration(duration) * time.Second)
 	defer ticker.Stop()
-	var totalCount, sealCount []int64
+	var totalCount, sealCount, delCount []int64
 	var err error
 	for range ticker.C {
 		latestBlockHeightAny := Cast(b.parserCtx.Indexer).GetLatestBlockHeight().Load()
@@ -417,22 +417,29 @@ func (b *BlockSyncerModular) dbStatistics(ctx context.Context) {
 				metrics.DataStatisticsErr.Inc()
 			}
 		}
-		totalCount, err = db.Cast(b.parserCtx.Database).GetObjectCount(false, latestBlockHeight)
+		totalCount, err = b.baseApp.GfBsDB().GetObjectCount(latestBlockHeight, "")
 		if err != nil {
 			metrics.DataStatisticsErr.Inc()
 			continue
 		}
-		sealCount, err = db.Cast(b.parserCtx.Database).GetObjectCount(true, latestBlockHeight)
+		sealCount, err = b.baseApp.GfBsDB().GetObjectCount(latestBlockHeight, "OBJECT_STATUS_SEALED")
+		if err != nil {
+			metrics.DataStatisticsErr.Inc()
+			continue
+		}
+		delCount, err = b.baseApp.GfBsDB().GetObjectCount(latestBlockHeight, "OBJECT_STATUS_DISCONTINUED")
 		if err != nil {
 			metrics.DataStatisticsErr.Inc()
 			continue
 		}
 		totalCountArr, _ := json.Marshal(totalCount)
 		sealCountArr, _ := json.Marshal(sealCount)
+		delCountArr, _ := json.Marshal(delCount)
 		err = db.Cast(b.parserCtx.Database).SaveStat(ctx, &models.DataStat{
 			BlockHeight:      latestBlockHeight,
 			ObjectSealCount:  string(sealCountArr),
 			ObjectTotalCount: string(totalCountArr),
+			ObjectDelCount:   string(delCountArr),
 			UpdateTime:       time.Now().Unix(),
 		})
 		if err != nil {

modular/blocksyncer/database/db_statistics.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"time"
 
-	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
-
 	"github.com/forbole/juno/v4/models"
 	"gorm.io/gorm/clause"
 
+	"github.com/bnb-chain/greenfield-storage-provider/pkg/log"
 	"github.com/bnb-chain/greenfield-storage-provider/store/bsdb"
 )
 

modular/gater/metadata_handler.go

@@ -2805,7 +2805,7 @@ func (g *GateModular) getBsDBDataInfoHandler(w http.ResponseWriter, r *http.Requ
 	ctx := context.Background()
 	queryParams := r.URL.Query()
 	blockHeightStr := queryParams.Get("block_height")
-	blockHeight, err = strconv.ParseInt(blockHeightStr, 10, 64)
+	blockHeight, err = util.StringToInt64(blockHeightStr)
 	if err != nil {
 		log.Errorw("failed parse block height", "error", err)
 		err = ErrInvalidQuery

modular/gater/router.go

@@ -264,7 +264,7 @@ func (g *GateModular) RegisterHandler(router *mux.Router) {
 	router.Path("/").Name(getBucketReadQuotaCountRouterName).Methods(http.MethodGet).Queries(ListBucketReadCountQuery, "").HandlerFunc(g.getBucketReadQuotaCountHandler)
 
 	// Get BsDB data statistics Info
-	router.Path("/").Name(getBsDBDataInfo).Methods(http.MethodGet).Queries(BsDBInfo, "").HandlerFunc(g.getBsDBDataInfoHandler)
+	router.Path("/").Name(getBsDBDataInfo).Methods(http.MethodGet).Queries(BsDBInfoQuery, "").HandlerFunc(g.getBsDBDataInfoHandler)
 
 	if g.env != gfspapp.EnvMainnet {
 		// Get Payment By Bucket ID

modular/gater/router_test.go

@@ -753,7 +753,7 @@ func TestRouters(t *testing.T) {
 			name:             "get block syncer data info",
 			router:           gwRouter,
 			method:           http.MethodGet,
-			url:              fmt.Sprintf("%s%s/?%s&%s", scheme, testDomain, BsDBInfo, "block_height"),
+			url:              fmt.Sprintf("%s%s/?%s&%s", scheme, testDomain, BsDBInfoQuery, "block_height"),
 			shouldMatch:      true,
 			wantedRouterName: getBsDBDataInfo,
 		},

store/bsdb/database.go

@@ -145,6 +145,8 @@ type Metadata interface {
 	GetDataMigrationRecordByProcessKey(processKey string) (*DataMigrationRecord, error)
 	// GetBsDBDataStatistics get the record of BsDB data statistics
 	GetBsDBDataStatistics(blockHeight uint64) (*DataStat, error)
+	// GetObjectCount get object count by update_at
+	GetObjectCount(blockHeight int64, objectStatus string) ([]int64, error)
 }
 
 // BSDB contains all the methods required by block syncer database

store/bsdb/object.go

@@ -525,3 +525,34 @@ func (b *BsDBImpl) GetBsDBDataStatistics(blockHeight uint64) (*DataStat, error)
 	}
 	return &dataRecord, err
 }
+
+func (b *BsDBImpl) GetObjectCount(blockHeight int64, objectStatus string) ([]int64, error) {
+	result := make([]int64, 0, ObjectsNumberOfShards)
+	step := int64(1000)
+	for i := 0; i < ObjectsNumberOfShards; i++ {
+		sum := int64(0)
+		primaryKey := int64(0)
+		count := int64(0)
+		for {
+			var err error
+			tmpDB := b.db.Table(GetObjectsTableNameByShardNumber(i))
+			if objectStatus != "" {
+				err = tmpDB.Where("id > ? and id <= ? and status = ? and update_at <= ?", primaryKey, primaryKey+step, objectStatus, blockHeight).Count(&count).Error
+			} else {
+				err = tmpDB.Where("id > ? and id <= ? and update_at <= ?", primaryKey, primaryKey+step, blockHeight).Count(&count).Error
+			}
+			if err == nil && count == 0 {
+				break
+			}
+			if err != nil {
+				log.Errorw("failed to get object count", "error", err, "left", primaryKey, "right", primaryKey+step)
+				return result, err
+			}
+			sum += count
+			primaryKey += step
+			time.Sleep(20 * time.Millisecond)
+		}
+		result = append(result, sum)
+	}
+	return result, nil
+}