Refactor submit domain (#9)

* Refactor submit domain. Add address submit domain.

* Extract a separate class to define address submit.

Author: Argeric

api/stat_api.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"encoding/json"
-	"strconv"
 
 	commonApi "github.com/Conflux-Chain/go-conflux-util/api"
 	"github.com/gin-gonic/gin"
@@ -20,7 +19,7 @@ const (
 	FeeStatType
 )
 
-func dashboard(c *gin.Context) (interface{}, error) {
+func dashboard(_ *gin.Context) (interface{}, error) {
 	submitStat, err := db.SubmitStatStore.LastByType(stat.Day)
 	if err != nil {
 		return nil, commonApi.ErrInternal(err)
@@ -31,7 +30,7 @@ func dashboard(c *gin.Context) (interface{}, error) {
 
 	storageBasicCost := StorageBasicCost{
 		TokenInfo:      *chargeToken,
-		BasicCostTotal: strconv.FormatUint(submitStat.BaseFeeTotal, 10),
+		BasicCostTotal: submitStat.BaseFeeTotal,
 	}
 	result := Dashboard{
 		StorageBasicCost: storageBasicCost,
@@ -92,33 +91,33 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
 	switch t {
 	case StorageStatType:
 		list := make([]DataStat, 0)
-		for _, stat := range *records {
+		for _, r := range *records {
 			list = append(list, DataStat{
-				StatTime:  &stat.StatTime,
-				FileCount: stat.FileCount,
-				FileTotal: stat.FileTotal,
-				DataSize:  stat.DataSize,
-				DataTotal: stat.DataTotal,
+				StatTime:  r.StatTime,
+				FileCount: r.FileCount,
+				FileTotal: r.FileTotal,
+				DataSize:  r.DataSize,
+				DataTotal: r.DataTotal,
 			})
 		}
 		result["list"] = list
 	case TxStatType:
 		list := make([]TxStat, 0)
-		for _, stat := range *records {
+		for _, r := range *records {
 			list = append(list, TxStat{
-				StatTime: &stat.StatTime,
-				TxCount:  stat.FileCount,
-				TxTotal:  stat.FileTotal,
+				StatTime: r.StatTime,
+				TxCount:  r.FileCount,
+				TxTotal:  r.FileTotal,
 			})
 		}
 		result["list"] = list
 	case FeeStatType:
 		list := make([]FeeStat, 0)
-		for _, stat := range *records {
+		for _, r := range *records {
 			list = append(list, FeeStat{
-				StatTime:     &stat.StatTime,
-				BaseFee:      stat.BaseFee,
-				BaseFeeTotal: stat.BaseFeeTotal,
+				StatTime:     r.StatTime,
+				BaseFee:      r.BaseFee,
+				BaseFeeTotal: r.BaseFeeTotal,
 			})
 		}
 		result["list"] = list

api/tx_api.go

@@ -3,15 +3,16 @@ package api
 import (
 	"encoding/hex"
 	"encoding/json"
+	"strconv"
+	"strings"
+
 	commonApi "github.com/Conflux-Chain/go-conflux-util/api"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/zero-gravity-labs/zerog-storage-scan/store"
 	"gorm.io/gorm"
-	"strconv"
-	"strings"
 )
 
 func listTx(c *gin.Context) (interface{}, error) {
@@ -61,12 +62,10 @@ func listTx(c *gin.Context) (interface{}, error) {
 			RootHash:  submit.RootHash,
 			Address:   addrMap[submit.SenderID].Address,
 			Method:    "submit",
+			Status:    submit.Status,
 			Timestamp: submit.BlockTime.Unix(),
 			DataSize:  submit.Length,
-			BaseFee:   submit.Value,
-		}
-		if submit.Finalized {
-			storageTx.Status = 1
+			BaseFee:   submit.Fee,
 		}
 		storageTxs = append(storageTxs, storageTx)
 	}
@@ -105,18 +104,16 @@ func getTxBrief(c *gin.Context) (interface{}, error) {
 		From:     addrMap[submit.SenderID].Address,
 		Method:   "submit",
 		RootHash: submit.RootHash,
+		Status:   submit.Status,
 		DataSize: submit.Length,
 		CostInfo: &CostInfo{
 			TokenInfo: *chargeToken,
-			BasicCost: submit.Value.String(),
+			BasicCost: submit.Fee,
 		},
 		BlockNumber: submit.BlockNumber,
 		TxHash:      submit.TxHash,
 		Timestamp:   uint64(submit.BlockTime.Unix()),
 	}
-	if submit.Finalized {
-		result.Status = 1
-	}
 
 	hash := common.HexToHash(submit.TxHash)
 	tx, err := sdk.Eth.TransactionByHash(hash)
@@ -158,7 +155,7 @@ func getTxDetail(c *gin.Context) (interface{}, error) {
 		return nil, errors.Errorf("Unmarshal submit extra error, txSeq %v", *param.TxSeq)
 	}
 
-	var nodes []SubmissionNode
+	nodes := make([]SubmissionNode, 0)
 	for _, n := range extra.Submission.Nodes {
 		nodes = append(nodes, SubmissionNode{
 			Root:   "0x" + hex.EncodeToString(n.Root[:]),

api/types.go

@@ -1,10 +1,11 @@
 package api
 
 import (
-	"github.com/shopspring/decimal"
 	"math/big"
 	"strings"
 	"time"
+
+	"github.com/shopspring/decimal"
 )
 
 type PageParam struct {
@@ -56,7 +57,7 @@ type TokenInfo struct {
 
 type CostInfo struct {
 	TokenInfo `json:"tokenInfo"`
-	BasicCost string `json:"basicCost"`
+	BasicCost decimal.Decimal `json:"basicCost"`
 }
 
 type SubmissionNode struct {
@@ -100,7 +101,7 @@ type TxDetail struct {
 
 type StorageBasicCost struct {
 	TokenInfo
-	BasicCostTotal string `json:"basicCostTotal"`
+	BasicCostTotal decimal.Decimal `json:"basicCostTotal"`
 }
 
 type Dashboard struct {
@@ -123,21 +124,21 @@ type FeeStatList struct {
 }
 
 type DataStat struct {
-	StatTime  *time.Time `json:"statTime"`
-	FileCount uint64     `json:"fileCount"`
-	FileTotal uint64     `json:"fileTotal"`
-	DataSize  uint64     `json:"dataSize"`
-	DataTotal uint64     `json:"dataTotal"`
+	StatTime  time.Time `json:"statTime"`
+	FileCount uint64    `json:"fileCount"`
+	FileTotal uint64    `json:"fileTotal"`
+	DataSize  uint64    `json:"dataSize"`
+	DataTotal uint64    `json:"dataTotal"`
 }
 
 type TxStat struct {
-	StatTime *time.Time `json:"statTime"`
-	TxCount  uint64     `json:"txCount"`
-	TxTotal  uint64     `json:"txTotal"`
+	StatTime time.Time `json:"statTime"`
+	TxCount  uint64    `json:"txCount"`
+	TxTotal  uint64    `json:"txTotal"`
 }
 
 type FeeStat struct {
-	StatTime     *time.Time `json:"statTime"`
-	BaseFee      uint64     `json:"baseFee"`
-	BaseFeeTotal uint64     `json:"baseFeeTotal"`
+	StatTime     time.Time       `json:"statTime"`
+	BaseFee      decimal.Decimal `json:"baseFee"`
+	BaseFeeTotal decimal.Decimal `json:"baseFeeTotal"`
 }

cmd/data_context.go

@@ -2,18 +2,19 @@ package cmd
 
 import (
 	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
 	"github.com/Conflux-Chain/go-conflux-util/store/mysql"
 	"github.com/Conflux-Chain/go-conflux-util/viper"
 	providers "github.com/openweb3/go-rpc-provider/provider_wrapper"
 	"github.com/openweb3/web3go"
 	"github.com/sirupsen/logrus"
 	"github.com/zero-gravity-labs/zerog-storage-client/node"
 	"github.com/zero-gravity-labs/zerog-storage-scan/store"
-	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
 )
 
 // DataContext context to hold sdk clients for blockchain interoperation.
@@ -44,6 +45,7 @@ var migrationModels = []interface{}{
 	&store.Block{},
 	&store.Config{},
 	&store.Submit{},
+	&store.AddressSubmit{},
 	&store.SubmitStat{},
 }
 

stat/stat_submit.go

@@ -102,7 +102,7 @@ func (ts *StatSubmit) statBasicRange(tr *TimeRange) (*store.SubmitStat, error) {
 		DataSize:     delta.DataSize,
 		DataTotal:    total.DataSize + delta.DataSize,
 		BaseFee:      delta.BaseFee,
-		BaseFeeTotal: total.BaseFee + delta.BaseFee,
+		BaseFeeTotal: total.BaseFee.Add(delta.BaseFee),
 	}, nil
 }
 
@@ -124,7 +124,7 @@ func (ts *StatSubmit) statRange(rangEnd *time.Time, srcStatType, descStatType st
 	if latestStat != nil {
 		srcStat.FileCount += latestStat.FileCount
 		srcStat.DataSize += latestStat.DataSize
-		srcStat.BaseFee += latestStat.BaseFee
+		srcStat.BaseFee = srcStat.BaseFee.Add(latestStat.BaseFee)
 	}
 
 	return &store.SubmitStat{
@@ -135,6 +135,6 @@ func (ts *StatSubmit) statRange(rangEnd *time.Time, srcStatType, descStatType st
 		DataSize:     srcStat.DataSize,
 		DataTotal:    destStat.DataSize + srcStat.DataSize,
 		BaseFee:      srcStat.BaseFee,
-		BaseFeeTotal: destStat.BaseFee + srcStat.BaseFee,
+		BaseFeeTotal: destStat.BaseFee.Add(srcStat.BaseFee),
 	}, nil
 }

store/store_address_submit.go

@@ -0,0 +1,25 @@
+package store
+
+import (
+	"time"
+
+	"github.com/shopspring/decimal"
+)
+
+type AddressSubmit struct {
+	SenderID        uint64 `gorm:"primary_key;autoIncrement:false"`
+	SubmissionIndex uint64 `gorm:"primary_key;autoIncrement:false"`
+	RootHash        string `gorm:"size:66;index:idx_root"`
+	Length          uint64 `gorm:"not null"`
+
+	BlockNumber uint64    `gorm:"not null"`
+	BlockTime   time.Time `gorm:"not null"`
+	TxHash      string    `gorm:"size:66;not null"`
+
+	Status uint64          `gorm:"not null;default:0"`
+	Fee    decimal.Decimal `gorm:"type:decimal(65);not null"`
+}
+
+func (AddressSubmit) TableName() string {
+	return "address_submits"
+}

store/store_block.go

@@ -19,7 +19,7 @@ func NewBlock(data *types.Block) *Block {
 	blockTime := time.Unix(int64(data.Timestamp), 0)
 	return &Block{
 		BlockNumber: data.Number.Uint64(),
-		Hash:        data.Hash.String()[2:],
+		Hash:        data.Hash.String(),
 		BlockTime:   blockTime,
 	}
 }

store/store_submit.go

@@ -14,24 +14,19 @@ import (
 	"gorm.io/gorm"
 )
 
-// TODO one table for overall submissions and another table for submissions of specific account.
-
 type Submit struct {
-	ID          uint64    `gorm:"index:idx_sender_id,priority:2"`
+	SubmissionIndex uint64 `gorm:"primaryKey;autoIncrement:false"`
+	RootHash        string `gorm:"size:66;index:idx_root"`
+	Sender          string `gorm:"-"`
+	SenderID        uint64 `gorm:"not null"`
+	Length          uint64 `gorm:"not null"`
+
 	BlockNumber uint64    `gorm:"not null"`
 	BlockTime   time.Time `gorm:"not null"`
 	TxHash      string    `gorm:"size:66;not null"`
 
-	// TODO use as primary key?
-	SubmissionIndex uint64 `gorm:"not null;index:idx_seq"`
-	RootHash        string `gorm:"size:66;index:idx_root"`
-	Sender          string `gorm:"-"`
-	SenderID        uint64 `gorm:"not null;index:idx_sender_id,priority:1"`
-	Length          uint64 `gorm:"not null"`
-	// TODO supports more status, including L1 and L2 status
-	Finalized bool `gorm:"default:false"`
-	// TODO change to Fee. how about use decimal(64,18) as sql type?
-	Value decimal.Decimal `gorm:"type:varchar(78);not null"`
+	Status uint64          `gorm:"not null;default:0"`
+	Fee    decimal.Decimal `gorm:"type:decimal(65);not null"`
 
 	Extra []byte `gorm:"type:mediumText"` // json field
 }
@@ -58,13 +53,13 @@ func NewSubmit(blockTime time.Time, log *types.Log, filter *contract.FlowFiltere
 	}
 
 	submit := &Submit{
-		BlockNumber:     log.BlockNumber,
-		BlockTime:       blockTime,
-		TxHash:          log.TxHash.String(),
 		SubmissionIndex: flowSubmit.SubmissionIndex.Uint64(),
 		Sender:          flowSubmit.Sender.String(),
 		Length:          flowSubmit.Submission.Length.Uint64(),
-		Value:           decimal.NewFromBigInt(big.NewInt(0), 0),
+		BlockNumber:     log.BlockNumber,
+		BlockTime:       blockTime,
+		TxHash:          log.TxHash.String(),
+		Fee:             decimal.NewFromBigInt(big.NewInt(0), 0),
 		Extra:           extra,
 	}
 
@@ -86,7 +81,27 @@ func newSubmitStore(db *gorm.DB) *SubmitStore {
 }
 
 func (ss *SubmitStore) Add(dbTx *gorm.DB, submits []*Submit) error {
-	return dbTx.CreateInBatches(submits, batchSizeInsert).Error
+	addressSubmits := make([]AddressSubmit, 0)
+	for _, submit := range submits {
+		addressSubmit := AddressSubmit{
+			SenderID:        submit.SenderID,
+			SubmissionIndex: submit.SubmissionIndex,
+			RootHash:        submit.RootHash,
+			Length:          submit.Length,
+			BlockNumber:     submit.BlockNumber,
+			BlockTime:       submit.BlockTime,
+			TxHash:          submit.TxHash,
+			Fee:             submit.Fee,
+			Status:          submit.Status,
+		}
+		addressSubmits = append(addressSubmits, addressSubmit)
+	}
+
+	if err := dbTx.CreateInBatches(submits, batchSizeInsert).Error; err != nil {
+		return err
+	}
+
+	return dbTx.CreateInBatches(addressSubmits, batchSizeInsert).Error
 }
 
 func (ss *SubmitStore) Pop(dbTx *gorm.DB, block uint64) error {
@@ -96,7 +111,7 @@ func (ss *SubmitStore) Pop(dbTx *gorm.DB, block uint64) error {
 func (ss *SubmitStore) Count(startTime, endTime time.Time) (*SubmitStatResult, error) {
 	var result SubmitStatResult
 	err := ss.DB.Model(&Submit{}).Select(`count(id) as file_count, IFNULL(sum(length), 0) as data_size, 
-		IFNULL(sum(value), 0) as base_fee`).Where("block_time >= ? and block_time < ?", startTime, endTime).
+		IFNULL(sum(fee), 0) as base_fee`).Where("block_time >= ? and block_time < ?", startTime, endTime).
 		Find(&result).Error
 	if err != nil {
 		return nil, err
@@ -119,8 +134,9 @@ func (ss *SubmitStore) FirstWithoutRootHash() (*Submit, error) {
 	return &submit, nil
 }
 
-func (ss *SubmitStore) Update(submit *Submit) error {
-	if err := ss.DB.Model(&submit).Updates(submit).Error; err != nil {
+func (ss *SubmitStore) UpdateByPrimaryKey(submit *Submit) error {
+	if err := ss.DB.Model(&submit).Where("submission_index=?", submit.SubmissionIndex).
+		Updates(submit).Error; err != nil {
 		return err
 	}