Refactor store coce

api/stat_api.go

@@ -2,13 +2,14 @@ package api
 
 import (
 	"encoding/json"
+	"strconv"
+
 	commonApi "github.com/Conflux-Chain/go-conflux-util/api"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/zero-gravity-labs/zerog-storage-scan/stat"
 	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 	"gorm.io/gorm"
-	"strconv"
 )
 
 type Type int
@@ -93,7 +94,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
 		list := make([]DataStat, 0)
 		for _, stat := range *records {
 			list = append(list, DataStat{
-				StatTime:  stat.StatTime,
+				StatTime:  &stat.StatTime,
 				FileCount: stat.FileCount,
 				FileTotal: stat.FileTotal,
 				DataSize:  stat.DataSize,
@@ -105,7 +106,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
 		list := make([]TxStat, 0)
 		for _, stat := range *records {
 			list = append(list, TxStat{
-				StatTime: stat.StatTime,
+				StatTime: &stat.StatTime,
 				TxCount:  stat.FileCount,
 				TxTotal:  stat.FileTotal,
 			})
@@ -115,7 +116,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
 		list := make([]FeeStat, 0)
 		for _, stat := range *records {
 			list = append(list, FeeStat{
-				StatTime:     stat.StatTime,
+				StatTime:     &stat.StatTime,
 				BaseFee:      stat.BaseFee,
 				BaseFeeTotal: stat.BaseFeeTotal,
 			})

stat/stat_submit.go

@@ -1,11 +1,12 @@
 package stat
 
 import (
+	"time"
+
 	"github.com/openweb3/web3go"
 	"github.com/pkg/errors"
 	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 	"gorm.io/gorm"
-	"time"
 )
 
 type StatSubmit struct {
@@ -94,7 +95,7 @@ func (ts *StatSubmit) statBasicRange(tr *TimeRange) (*store.SubmitStat, error) {
 	}
 
 	return &store.SubmitStat{
-		StatTime:     tr.start,
+		StatTime:     *tr.start,
 		StatType:     ts.statType,
 		FileCount:    delta.FileCount,
 		FileTotal:    total.FileCount + delta.FileCount,
@@ -127,7 +128,7 @@ func (ts *StatSubmit) statRange(rangEnd *time.Time, srcStatType, descStatType st
 	}
 
 	return &store.SubmitStat{
-		StatTime:     rangeStart,
+		StatTime:     *rangeStart,
 		StatType:     descStatType,
 		FileCount:    srcStat.FileCount,
 		FileTotal:    destStat.FileCount + srcStat.FileCount,

store/store_address.go

@@ -1,9 +1,10 @@
 package store
 
 import (
+	"time"
+
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
 	"gorm.io/gorm"
-	"time"
 )
 
 type Address struct {
@@ -26,7 +27,7 @@ func newAddressStore(db *gorm.DB) *AddressStore {
 	}
 }
 
-func (as *AddressStore) Add(dbTx *gorm.DB, address string, blockTime time.Time) (uint64, error) {
+func (as *AddressStore) Add(address string, blockTime time.Time) (uint64, error) {
 	var addr Address
 	existed, err := as.Store.Exists(&addr, "address = ?", address) //TODO using LRU cache for improving the query performance
 	if err != nil {
@@ -40,10 +41,8 @@ func (as *AddressStore) Add(dbTx *gorm.DB, address string, blockTime time.Time)
 		Address:   address,
 		BlockTime: blockTime,
 	}
-	if dbTx == nil {
-		dbTx = as.Store.DB
-	}
-	if err := dbTx.Create(&addr).Error; err != nil {
+
+	if err := as.DB.Create(&addr).Error; err != nil {
 		return 0, err
 	}
 

store/store_block.go

@@ -2,16 +2,17 @@ package store
 
 import (
 	"database/sql"
+	"time"
+
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
 	"github.com/openweb3/web3go/types"
 	"gorm.io/gorm"
-	"time"
 )
 
 type Block struct {
 	BlockNumber uint64    `gorm:"primaryKey;autoIncrement:false"`
 	Hash        string    `gorm:"size:66;not null"`
-	BlockTime   time.Time `gorm:"not null;index:idx_block_time,sort:desc"`
+	BlockTime   time.Time `gorm:"not null;index:idx_block_time"`
 }
 
 func NewBlock(data *types.Block) *Block {

store/store_submit.go

@@ -2,30 +2,36 @@ package store
 
 import (
 	"encoding/json"
+	"math/big"
+	"time"
+
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/openweb3/web3go/types"
 	"github.com/pkg/errors"
 	"github.com/shopspring/decimal"
 	"github.com/zero-gravity-labs/zerog-storage-client/contract"
 	"gorm.io/gorm"
-	"math/big"
-	"time"
 )
 
+// TODO one table for overall submissions and another table for submissions of specific account.
+
 type Submit struct {
 	ID          uint64    `gorm:"index:idx_sender_id,priority:2"`
-	BlockNumber uint64    `gorm:"not null;index:idx_bn"`
-	BlockTime   time.Time `gorm:"not null;index:idx_blockTime,sort:desc"`
-	TxHash      string    `gorm:"size:66;not null;index:idx_hash"`
-
-	SubmissionIndex uint64          `gorm:"not null"`
-	RootHash        string          `gorm:"size:66;index:idx_root"`
-	Sender          string          `gorm:"-"`
-	SenderID        uint64          `gorm:"not null;index:idx_sender_id,priority:1"`
-	Length          uint64          `gorm:"not null"`
-	Finalized       bool            `gorm:"default:false"`
-	Value           decimal.Decimal `gorm:"type:varchar(78);not null"`
+	BlockNumber uint64    `gorm:"not null"`
+	BlockTime   time.Time `gorm:"not null"`
+	TxHash      string    `gorm:"size:66;not null"`
+
+	// TODO use as primary key?
+	SubmissionIndex uint64 `gorm:"not null;index:idx_seq"`
+	RootHash        string `gorm:"size:66;index:idx_root"`
+	Sender          string `gorm:"-"`
+	SenderID        uint64 `gorm:"not null;index:idx_sender_id,priority:1"`
+	Length          uint64 `gorm:"not null"`
+	// TODO supports more status, including L1 and L2 status
+	Finalized bool `gorm:"default:false"`
+	// TODO change to Fee. how about use decimal(64,18) as sql type?
+	Value decimal.Decimal `gorm:"type:varchar(78);not null"`
 
 	Extra []byte `gorm:"type:mediumText"` // json field
 }

store/store_submit_stat.go

@@ -1,22 +1,24 @@
 package store
 
 import (
+	"time"
+
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
 	"github.com/pkg/errors"
 	"gorm.io/gorm"
-	"time"
 )
 
 type SubmitStat struct {
-	ID           uint64     `gorm:"primaryKey" json:"-"`
-	StatTime     *time.Time `gorm:"not null;index:idx_statTime_statType,unique,priority:1" json:"statTime"`
-	StatType     string     `gorm:"type:char(3);not null;index:idx_statTime_statType,unique,priority:2" json:"-"`
-	FileCount    uint64     `gorm:"not null;default:0" json:"fileCount"`    // Number of files in a specific time interval
-	FileTotal    uint64     `gorm:"not null;default:0" json:"fileTotal"`    // Total number of files by a certain time
-	DataSize     uint64     `gorm:"not null;default:0" json:"dataSize"`     // Size of storage data in a specific time interval
-	DataTotal    uint64     `gorm:"not null;default:0" json:"dataTotal"`    // Total Size of storage data by a certain time
-	BaseFee      uint64     `gorm:"not null;default:0" json:"baseFee"`      // The base fee for storage
-	BaseFeeTotal uint64     `gorm:"not null;default:0" json:"baseFeeTotal"` // The total base fee for storage
+	ID        uint64    `json:"-"`
+	StatType  string    `gorm:"size:4;not null;uniqueIndex:idx_statType_statTime,priority:1" json:"-"`
+	StatTime  time.Time `gorm:"not null;uniqueIndex:idx_statType_statTime,priority:2" json:"statTime"`
+	FileCount uint64    `gorm:"not null;default:0" json:"fileCount"` // Number of files in a specific time interval
+	FileTotal uint64    `gorm:"not null;default:0" json:"fileTotal"` // Total number of files by a certain time
+	DataSize  uint64    `gorm:"not null;default:0" json:"dataSize"`  // Size of storage data in a specific time interval
+	DataTotal uint64    `gorm:"not null;default:0" json:"dataTotal"` // Total Size of storage data by a certain time
+	// TODO not enough for blockchain value of decimals 18.
+	BaseFee      uint64 `gorm:"not null;default:0" json:"baseFee"`      // The base fee for storage
+	BaseFeeTotal uint64 `gorm:"not null;default:0" json:"baseFeeTotal"` // The total base fee for storage
 }
 
 func (SubmitStat) TableName() string {
@@ -35,7 +37,7 @@ func newSubmitStatStore(db *gorm.DB) *SubmitStatStore {
 
 func (t *SubmitStatStore) LastByType(statType string) (*SubmitStat, error) {
 	var submitStat SubmitStat
-	err := t.Store.DB.Where("stat_type = ?", statType).Last(&submitStat).Error
+	err := t.Store.DB.Where("stat_type = ?", statType).Order("stat_time asc").Last(&submitStat).Error
 	if errors.Is(err, gorm.ErrRecordNotFound) {
 		return nil, nil
 	}
@@ -59,13 +61,13 @@ func (t *SubmitStatStore) Sum(startTime, endTime *time.Time, statType string) (*
 	db := t.DB.Model(&SubmitStat{}).Select(`IFNULL(sum(file_count), 0) as file_count, 
 		IFNULL(sum(data_size), 0) as data_size, IFNULL(sum(base_fee), 0) as base_fee`)
 	if startTime != nil && endTime != nil {
-		db = db.Where("stat_time >= ? and stat_time < ? and stat_type = ?", startTime, endTime, statType)
+		db = db.Where("stat_type = ? and stat_time >= ? and stat_time < ?", statType, startTime, endTime)
 	}
 	if startTime != nil && endTime == nil {
-		db = db.Where("stat_time >= ? and stat_type = ?", startTime, statType)
+		db = db.Where("stat_type = ? and stat_time >= ?", statType, startTime)
 	}
 	if startTime == nil && endTime != nil {
-		db = db.Where("stat_time < ? and stat_type = ?", endTime, statType)
+		db = db.Where("stat_type = ? and stat_time < ?", statType, endTime)
 	}
 
 	var sum SubmitStatResult
@@ -82,5 +84,5 @@ func (t *SubmitStatStore) Add(dbTx *gorm.DB, submitStat []*SubmitStat) error {
 }
 
 func (t *SubmitStatStore) Del(dbTx *gorm.DB, submitStat *SubmitStat) error {
-	return dbTx.Where("stat_time = ? and stat_type = ?", submitStat.StatTime, submitStat.StatType).Delete(&SubmitStat{}).Error
+	return dbTx.Where("stat_type = ? and stat_time = ?", submitStat.StatType, submitStat.StatTime).Delete(&SubmitStat{}).Error
 }

sync/blockchain.go

@@ -2,25 +2,32 @@ package sync
 
 import (
 	"context"
+
 	set "github.com/deckarep/golang-set"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/openweb3/go-rpc-provider"
 	"github.com/openweb3/web3go"
 	"github.com/openweb3/web3go/types"
 	"github.com/pkg/errors"
-	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 )
 
 var (
 	ErrNotFound     = errors.New("not found")
 	ErrChainReorged = errors.New("chain re-orged")
 )
 
+type EthData struct {
+	Number   uint64
+	Block    *types.Block
+	Receipts map[common.Hash]*types.Receipt
+	Logs     []types.Log
+}
+
 func isTxExecutedInBlock(tx *types.TransactionDetail, receipt *types.Receipt) bool {
 	return tx != nil && receipt.Status != nil && *receipt.Status < 2
 }
 
-func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthData, error) {
+func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*EthData, error) {
 	// get block
 	block, err := w3c.Eth.BlockByNumber(types.BlockNumber(blockNumber), true)
 	if err != nil {
@@ -48,13 +55,10 @@ func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthDat
 	}
 	for i := 0; i < len(blockTxs); i++ {
 		tx := blockTxs[i]
-		var receipt *types.Receipt
-		receipt = &blockReceipts[i]
+		receipt := &blockReceipts[i]
 
 		// check re-org
 		switch {
-		case receipt == nil: // receipt shouldn't be nil unless chain re-org
-			return nil, errors.WithMessage(ErrChainReorged, "tx receipt nil")
 		case receipt.BlockHash != block.Hash:
 			return nil, errors.WithMessagef(ErrChainReorged, "receipt block hash mismatch, rcptBlkHash %v, blkHash %v",
 				receipt.BlockHash, block.Hash)
@@ -68,10 +72,10 @@ func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthDat
 		txReceipts[tx.Hash] = receipt
 	}
 
-	return &store.EthData{Number: blockNumber, Block: block, Receipts: txReceipts}, nil
+	return &EthData{Number: blockNumber, Block: block, Receipts: txReceipts}, nil
 }
 
-func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Address, flowSubmitSig common.Hash) (*store.EthData, error) {
+func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Address, flowSubmitSig common.Hash) (*EthData, error) {
 	// get block
 	block, err := w3c.Eth.BlockByNumber(types.BlockNumber(blockNumber), true)
 	if err != nil {
@@ -111,7 +115,7 @@ func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Ad
 		logs = append(logs, log)
 	}
 
-	return &store.EthData{Number: blockNumber, Block: block, Logs: logs}, nil
+	return &EthData{Number: blockNumber, Block: block, Logs: logs}, nil
 }
 
 func batchGetFlowSubmits(w3c *web3go.Client, blockFrom, blockTo uint64, flowAddr common.Address,

sync/catchup.go

@@ -2,6 +2,10 @@ package sync
 
 import (
 	"context"
+	"math/big"
+	"strings"
+	"time"
+
 	viperutil "github.com/Conflux-Chain/go-conflux-util/viper"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/openweb3/web3go"
@@ -10,9 +14,6 @@ import (
 	"github.com/sirupsen/logrus"
 	nhContract "github.com/zero-gravity-labs/zerog-storage-scan/contract"
 	"github.com/zero-gravity-labs/zerog-storage-scan/store"
-	"math/big"
-	"strings"
-	"time"
 )
 
 type CatchupSyncer struct {
@@ -215,7 +216,7 @@ func (s *CatchupSyncer) convertSubmits(logs []types.Log, blockNum2TimeMap map[ui
 			return nil, err
 		}
 
-		senderId, err := s.db.AddressStore.Add(nil, submit.Sender, blockTime)
+		senderId, err := s.db.AddressStore.Add(submit.Sender, blockTime)
 		if err != nil {
 			return nil, err
 		}