Refactor db domains (#5)

* Refactor db domains including block, submit and submitstat.

* Rename basiccost to basefee.

Author: Argeric

api/tx_api.go

@@ -46,7 +46,7 @@ func listTx(c *gin.Context) (interface{}, error) {
 	addrIds := make([]uint64, 0)
 	txHashes := make([]string, 0)
 	for _, submit := range *submits {
-		addrIds = append(addrIds, submit.SenderId)
+		addrIds = append(addrIds, submit.SenderID)
 		txHashes = append(txHashes, submit.TxHash)
 	}
 	addrMap, err := db.BatchGetAddresses(addrIds)
@@ -61,7 +61,7 @@ func listTx(c *gin.Context) (interface{}, error) {
 			BlockNum: submit.BlockNumber,
 			TxHash:   "0x" + submit.TxHash,
 			RootHash: "0x" + submit.RootHash,
-			Address:  addrMap[submit.SenderId].Address,
+			Address:  addrMap[submit.SenderID].Address,
 			Method:   "submit",
 		}
 		storageTxs = append(storageTxs, storageTx)
@@ -136,6 +136,7 @@ func getTxBrief(c *gin.Context) (interface{}, error) {
 	return result, nil
 }
 
+// TODO add StartPos, EndPos and PieceCounts when refactor api module
 func getTxDetail(c *gin.Context) (interface{}, error) {
 	var param queryTxParam
 	if err := c.ShouldBind(&param); err != nil {
@@ -158,12 +159,12 @@ func getTxDetail(c *gin.Context) (interface{}, error) {
 	}
 
 	result := TxDetail{
-		TxSeq:       strconv.FormatUint(submit.SubmissionIndex, 10),
-		RootHash:    "0x" + submit.RootHash,
-		StartPos:    submit.StartPos,
-		EndPos:      submit.StartPos + submit.Length,
-		PieceCounts: submit.Nodes,
-		Pieces:      nodes,
+		TxSeq:    strconv.FormatUint(submit.SubmissionIndex, 10),
+		RootHash: "0x" + submit.RootHash,
+		//StartPos:    submit.StartPos,
+		//EndPos:      submit.StartPos + submit.Length,
+		//PieceCounts: submit.Nodes,
+		Pieces: nodes,
 	}
 
 	return result, nil

stat/stat.go

@@ -3,11 +3,11 @@ package stat
 import (
 	"context"
 	"github.com/Conflux-Chain/go-conflux-util/viper"
-	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 	"github.com/openweb3/web3go"
 	"github.com/openweb3/web3go/types"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 	"sync"
 	"time"
 )

stat/stat_submit.go

@@ -83,54 +83,57 @@ func (ts *StatSubmit) calculateStat(tr *TimeRange) error {
 	})
 }
 
-// TODO add stat for cost and cost total when refactor db domains
 func (ts *StatSubmit) statBasicRange(tr *TimeRange) (*store.SubmitStat, error) {
-	fileCount, dataSize, err := ts.Db.SubmitStore.Count(tr.start, tr.end)
+	delta, err := ts.Db.SubmitStore.Count(*tr.start, *tr.end)
 	if err != nil {
 		return nil, err
 	}
-	fileTotal, dataTotal, err := ts.Db.SubmitStatStore.Sum(nil, tr.start, ts.statType)
+	total, err := ts.Db.SubmitStatStore.Sum(nil, tr.start, ts.statType)
 	if err != nil {
 		return nil, err
 	}
 
 	return &store.SubmitStat{
-		StatTime:  tr.start,
-		StatType:  ts.statType,
-		FileCount: fileCount,
-		FileTotal: fileTotal + fileCount,
-		DataSize:  dataSize,
-		DataTotal: dataTotal + dataSize,
+		StatTime:     tr.start,
+		StatType:     ts.statType,
+		FileCount:    delta.FileCount,
+		FileTotal:    total.FileCount + delta.FileCount,
+		DataSize:     delta.DataSize,
+		DataTotal:    total.DataSize + delta.DataSize,
+		BaseFee:      delta.BaseFee,
+		BaseFeeTotal: total.BaseFee + delta.BaseFee,
 	}, nil
 }
 
-// TODO add stat for cost and cost total when refactor db domains
 func (ts *StatSubmit) statRange(rangEnd *time.Time, srcStatType, descStatType string, latestStat *store.SubmitStat) (*store.SubmitStat, error) {
 	rangeStart, err := ts.calStatRangeStart(rangEnd, descStatType)
 	if err != nil {
 		return nil, err
 	}
 
-	fileCount, dataSize, err := ts.Db.SubmitStatStore.Sum(rangeStart, rangEnd, srcStatType)
+	srcStat, err := ts.Db.SubmitStatStore.Sum(rangeStart, rangEnd, srcStatType)
 	if err != nil {
 		return nil, err
 	}
-	fileTotal, dataTotal, err := ts.Db.SubmitStatStore.Sum(nil, rangeStart, descStatType)
+	destStat, err := ts.Db.SubmitStatStore.Sum(nil, rangeStart, descStatType)
 	if err != nil {
 		return nil, err
 	}
 
 	if latestStat != nil {
-		fileCount += latestStat.FileCount
-		dataSize += latestStat.DataSize
+		srcStat.FileCount += latestStat.FileCount
+		srcStat.DataSize += latestStat.DataSize
+		srcStat.BaseFee += latestStat.BaseFee
 	}
 
 	return &store.SubmitStat{
-		StatTime:  rangeStart,
-		StatType:  descStatType,
-		FileCount: fileCount,
-		FileTotal: fileTotal + fileCount,
-		DataSize:  dataSize,
-		DataTotal: dataTotal + dataSize,
+		StatTime:     rangeStart,
+		StatType:     descStatType,
+		FileCount:    srcStat.FileCount,
+		FileTotal:    destStat.FileCount + srcStat.FileCount,
+		DataSize:     srcStat.DataSize,
+		DataTotal:    destStat.DataSize + srcStat.DataSize,
+		BaseFee:      srcStat.BaseFee,
+		BaseFeeTotal: destStat.BaseFee + srcStat.BaseFee,
 	}, nil
 }

store/store.go

@@ -14,24 +14,23 @@ type MysqlStore struct {
 	*mysql.Store
 	*AddressStore
 	*BlockStore
+	*ConfigStore
 	*SubmitStore
 	*SubmitStatStore
-	*ConfigStore
 }
 
 func MustNewStore(db *gorm.DB) *MysqlStore {
 	return &MysqlStore{
 		Store:           mysql.NewStore(db),
 		AddressStore:    newAddressStore(db),
 		BlockStore:      newBlockStore(db),
+		ConfigStore:     newConfigStore(db),
 		SubmitStore:     newSubmitStore(db),
 		SubmitStatStore: newSubmitStatStore(db),
-		ConfigStore:     newConfigStore(db),
 	}
 }
 
 func (ms *MysqlStore) Push(block *Block, submits []*Submit) error {
-
 	return ms.Store.DB.Transaction(func(dbTx *gorm.DB) error {
 		// save blocks
 		if err := ms.BlockStore.Add(dbTx, block); err != nil {

store/store_block.go

@@ -70,7 +70,7 @@ func (bs *BlockStore) BlockHash(blockNumber uint64) (string, bool, error) {
 func (bs *BlockStore) FirstBlockAfterTime(t *time.Time) (uint64, bool, error) {
 	var blk Block
 
-	result := bs.DB.Where("block_time >= ?", t).Order("block_time desc").Limit(1).Find(&blk)
+	result := bs.DB.Where("block_time >= ?", t).Order("block_time asc").Limit(1).Find(&blk)
 	if result.Error != nil {
 		return 0, false, result.Error
 	}

store/store_config.go

@@ -6,10 +6,6 @@ import (
 	"gorm.io/gorm/clause"
 )
 
-const (
-	CfgDataUplinkRate = "dataUplinkRate"
-)
-
 type Config struct {
 	Name  string `gorm:"size:32;primaryKey"`
 	Value string `gorm:"size:512"`

store/store_submit.go

@@ -1,48 +1,65 @@
 package store
 
 import (
-	"encoding/hex"
+	"encoding/json"
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
-	"github.com/zero-gravity-labs/zerog-storage-client/contract"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/openweb3/web3go/types"
 	"github.com/pkg/errors"
+	"github.com/shopspring/decimal"
+	"github.com/zero-gravity-labs/zerog-storage-client/contract"
 	"gorm.io/gorm"
+	"math/big"
 	"time"
 )
 
 type Submit struct {
-	ID               uint64     `gorm:"primaryKey;index:idx_sender_id,priority:2"`
-	BlockNumber      uint64     `gorm:"not null;index:idx_bn"`
-	TxHash           string     `gorm:"type:varchar(64);not null;index:idx_hash,length:10"`
-	CreatedAt        *time.Time `gorm:"not null;index:idx_createdAt,sort:desc"`
-	Sender           string     `gorm:"-"`
-	SenderId         uint64     `gorm:"not null;index:idx_sender_id,priority:1"`
-	RootHash         string     `gorm:"size:64;index:idx_root,length:10"`
-	Identity         string     `gorm:"size:64;not null"`
-	SubmissionIndex  uint64     `gorm:"not null"`
-	StartPos         uint64     `gorm:"not null"`
-	Length           uint64     `gorm:"not null"`
-	SubmissionLength uint64     `gorm:"not null"`
-	Nodes            uint64     `gorm:"not null"`
+	ID          uint64    `gorm:"index:idx_sender_id,priority:2"`
+	BlockNumber uint64    `gorm:"not null;index:idx_bn"`
+	BlockTime   time.Time `gorm:"not null;index:idx_blockTime,sort:desc"`
+	TxHash      string    `gorm:"size:66;not null;index:idx_hash"`
+
+	SubmissionIndex uint64          `gorm:"not null"`
+	RootHash        string          `gorm:"size:66;index:idx_root"`
+	Sender          string          `gorm:"-"`
+	SenderID        uint64          `gorm:"not null;index:idx_sender_id,priority:1"`
+	Length          uint64          `gorm:"not null"`
+	Finalized       bool            `gorm:"default:false"`
+	Value           decimal.Decimal `gorm:"type:varchar(78);not null"`
+
+	Extra []byte `gorm:"type:mediumText"` // json field
 }
 
-func NewSubmit(blockTime *time.Time, log *types.Log, filter *contract.FlowFilterer) (*Submit, error) {
+type SubmitExtra struct {
+	Identity   common.Hash         `json:"identity"`
+	StartPos   *big.Int            `json:"startPos"`
+	Submission contract.Submission `json:"submission"`
+}
+
+func NewSubmit(blockTime time.Time, log *types.Log, filter *contract.FlowFilterer) (*Submit, error) {
 	flowSubmit, err := filter.ParseSubmit(*log.ToEthLog())
 	if err != nil {
 		return nil, err
 	}
 
+	extra, err := json.Marshal(SubmitExtra{
+		Identity:   flowSubmit.Identity,
+		StartPos:   flowSubmit.StartPos,
+		Submission: flowSubmit.Submission,
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	submit := &Submit{
-		BlockNumber:      log.BlockNumber,
-		TxHash:           log.TxHash.String()[2:],
-		CreatedAt:        blockTime,
-		Sender:           flowSubmit.Sender.String()[2:],
-		Identity:         hex.EncodeToString(flowSubmit.Identity[:]),
-		SubmissionIndex:  flowSubmit.SubmissionIndex.Uint64(),
-		StartPos:         flowSubmit.StartPos.Uint64(),
-		Length:           flowSubmit.Length.Uint64(),
-		SubmissionLength: flowSubmit.Submission.Length.Uint64(),
-		Nodes:            uint64(len(flowSubmit.Submission.Nodes)),
+		BlockNumber:     log.BlockNumber,
+		BlockTime:       blockTime,
+		TxHash:          log.TxHash.String(),
+		SubmissionIndex: flowSubmit.SubmissionIndex.Uint64(),
+		Sender:          flowSubmit.Sender.String(),
+		Length:          flowSubmit.Submission.Length.Uint64(),
+		Value:           decimal.NewFromBigInt(big.NewInt(0), 0),
+		Extra:           extra,
 	}
 
 	return submit, nil
@@ -70,18 +87,16 @@ func (ss *SubmitStore) Pop(dbTx *gorm.DB, block uint64) error {
 	return dbTx.Where("block_number >= ?", block).Delete(&Submit{}).Error
 }
 
-func (ss *SubmitStore) Count(startTime, endTime *time.Time) (uint64, uint64, error) {
-	var result struct {
-		FileCount int64
-		DataSize  int64
-	}
-	err := ss.DB.Model(&Submit{}).Select("count(id) as file_count, IFNULL(sum(submission_length), 0) as data_size").
-		Where("created_at >= ? and created_at < ?", startTime, endTime).Find(&result).Error
+func (ss *SubmitStore) Count(startTime, endTime time.Time) (*SubmitStatResult, error) {
+	var result SubmitStatResult
+	err := ss.DB.Model(&Submit{}).Select(`count(id) as file_count, IFNULL(sum(length), 0) as data_size, 
+		IFNULL(sum(value), 0) as base_fee`).Where("block_time >= ? and block_time < ?", startTime, endTime).
+		Find(&result).Error
 	if err != nil {
-		return 0, 0, err
+		return nil, err
 	}
 
-	return uint64(result.FileCount), uint64(result.DataSize), nil
+	return &result, nil
 }
 
 func (ss *SubmitStore) FirstWithoutRootHash() (*Submit, error) {

store/store_submit_stat.go

@@ -8,13 +8,15 @@ import (
 )
 
 type SubmitStat struct {
-	ID        uint64     `gorm:"primaryKey" json:"-"`
-	StatTime  *time.Time `gorm:"not null;index:idx_statTime_statType,unique,priority:1" json:"statTime"`
-	StatType  string     `gorm:"type:char(3);not null;index:idx_statTime_statType,unique,priority:2" json:"-"`
-	FileCount uint64     `gorm:"not null;default:0" json:"fileCount"` // Number of files in a specific time interval
-	FileTotal uint64     `gorm:"not null;default:0" json:"fileTotal"` // Total number of files by a certain time
-	DataSize  uint64     `gorm:"not null;default:0" json:"dataSize"`  // Size of storage data in a specific time interval
-	DataTotal uint64     `gorm:"not null;default:0" json:"dataTotal"` // Total Size of storage data by a certain time
+	ID           uint64     `gorm:"primaryKey" json:"-"`
+	StatTime     *time.Time `gorm:"not null;index:idx_statTime_statType,unique,priority:1" json:"statTime"`
+	StatType     string     `gorm:"type:char(3);not null;index:idx_statTime_statType,unique,priority:2" json:"-"`
+	FileCount    uint64     `gorm:"not null;default:0" json:"fileCount"`    // Number of files in a specific time interval
+	FileTotal    uint64     `gorm:"not null;default:0" json:"fileTotal"`    // Total number of files by a certain time
+	DataSize     uint64     `gorm:"not null;default:0" json:"dataSize"`     // Size of storage data in a specific time interval
+	DataTotal    uint64     `gorm:"not null;default:0" json:"dataTotal"`    // Total Size of storage data by a certain time
+	BaseFee      uint64     `gorm:"not null;default:0" json:"baseFee"`      // The basic cost for storage
+	BaseFeeTotal uint64     `gorm:"not null;default:0" json:"baseFeeTotal"` // The total basic cost for storage
 }
 
 func (SubmitStat) TableName() string {
@@ -43,12 +45,19 @@ func (t *SubmitStatStore) LastByType(statType string) (*SubmitStat, error) {
 	return &submitStat, nil
 }
 
-func (t *SubmitStatStore) Sum(startTime, endTime *time.Time, statType string) (uint64, uint64, error) {
+type SubmitStatResult struct {
+	FileCount uint64
+	DataSize  uint64
+	BaseFee   uint64
+}
+
+func (t *SubmitStatStore) Sum(startTime, endTime *time.Time, statType string) (*SubmitStatResult, error) {
 	if startTime == nil && endTime == nil {
-		return 0, 0, errors.New("At least provide one parameter for startTime and endTime")
+		return nil, errors.New("At least provide one parameter for startTime and endTime")
 	}
 
-	db := t.DB.Model(&SubmitStat{}).Select("IFNULL(sum(file_count), 0) as file_sum, IFNULL(sum(data_size), 0) as data_sum")
+	db := t.DB.Model(&SubmitStat{}).Select(`IFNULL(sum(file_count), 0) as file_count, 
+		IFNULL(sum(data_size), 0) as data_size, IFNULL(sum(base_fee), 0) as base_fee`)
 	if startTime != nil && endTime != nil {
 		db = db.Where("stat_time >= ? and stat_time < ? and stat_type = ?", startTime, endTime, statType)
 	}
@@ -59,17 +68,13 @@ func (t *SubmitStatStore) Sum(startTime, endTime *time.Time, statType string) (u
 		db = db.Where("stat_time < ? and stat_type = ?", endTime, statType)
 	}
 
-	var sum struct {
-		FileSum int64
-		DataSum int64
-	}
-
+	var sum SubmitStatResult
 	err := db.Find(&sum).Error
 	if err != nil {
-		return 0, 0, err
+		return nil, err
 	}
 
-	return uint64(sum.FileSum), uint64(sum.DataSum), nil
+	return &sum, nil
 }
 
 func (t *SubmitStatStore) Add(dbTx *gorm.DB, submitStat []*SubmitStat) error {

sync/catchup.go

@@ -210,7 +210,7 @@ func (s *CatchupSyncer) convertSubmits(logs []types.Log, blockNum2TimeMap map[ui
 	for _, log := range logs {
 		ts := blockNum2TimeMap[log.BlockNumber]
 		blockTime := time.Unix(int64(ts), 0)
-		submit, err := store.NewSubmit(&blockTime, &log, nhContract.DummyFlowFilterer())
+		submit, err := store.NewSubmit(blockTime, &log, nhContract.DummyFlowFilterer())
 		if err != nil {
 			return nil, err
 		}
@@ -220,7 +220,7 @@ func (s *CatchupSyncer) convertSubmits(logs []types.Log, blockNum2TimeMap map[ui
 			return nil, err
 		}
 
-		submit.SenderId = senderId
+		submit.SenderID = senderId
 		submits = append(submits, submit)
 	}