Skip to content

Commit d586dae

Browse files
boqiuboqiu
authored
Refactor store coce (#8)
* Move EthData struct from store to sync package * refine gorm declarations * refactor address store * update block store index * refactor submit_stat store * aaa * refactor submit store
1 parent 1d99bb7 commit d586dae

File tree

10 files changed

+78
-75
lines changed

10 files changed

+78
-75
lines changed

‎api/stat_api.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ package api
22

33
import (
44
"encoding/json"
5+
"strconv"
6+
57
commonApi "github.com/Conflux-Chain/go-conflux-util/api"
68
"github.com/gin-gonic/gin"
79
"github.com/sirupsen/logrus"
810
"github.com/zero-gravity-labs/zerog-storage-scan/stat"
911
"github.com/zero-gravity-labs/zerog-storage-scan/store"
1012
"gorm.io/gorm"
11-
"strconv"
1213
)
1314

1415
type Type int
@@ -93,7 +94,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
9394
list := make([]DataStat, 0)
9495
for _, stat := range *records {
9596
list = append(list, DataStat{
96-
StatTime: stat.StatTime,
97+
StatTime: &stat.StatTime,
9798
FileCount: stat.FileCount,
9899
FileTotal: stat.FileTotal,
99100
DataSize: stat.DataSize,
@@ -105,7 +106,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
105106
list := make([]TxStat, 0)
106107
for _, stat := range *records {
107108
list = append(list, TxStat{
108-
StatTime: stat.StatTime,
109+
StatTime: &stat.StatTime,
109110
TxCount: stat.FileCount,
110111
TxTotal: stat.FileTotal,
111112
})
@@ -115,7 +116,7 @@ func getSubmitStatByType(c *gin.Context, t Type) (interface{}, error) {
115116
list := make([]FeeStat, 0)
116117
for _, stat := range *records {
117118
list = append(list, FeeStat{
118-
StatTime: stat.StatTime,
119+
StatTime: &stat.StatTime,
119120
BaseFee: stat.BaseFee,
120121
BaseFeeTotal: stat.BaseFeeTotal,
121122
})

‎stat/stat_submit.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
package stat
22

33
import (
4+
"time"
5+
46
"github.com/openweb3/web3go"
57
"github.com/pkg/errors"
68
"github.com/zero-gravity-labs/zerog-storage-scan/store"
79
"gorm.io/gorm"
8-
"time"
910
)
1011

1112
type StatSubmit struct {
@@ -94,7 +95,7 @@ func (ts *StatSubmit) statBasicRange(tr *TimeRange) (*store.SubmitStat, error) {
9495
}
9596

9697
return &store.SubmitStat{
97-
StatTime: tr.start,
98+
StatTime: *tr.start,
9899
StatType: ts.statType,
99100
FileCount: delta.FileCount,
100101
FileTotal: total.FileCount + delta.FileCount,
@@ -127,7 +128,7 @@ func (ts *StatSubmit) statRange(rangEnd *time.Time, srcStatType, descStatType st
127128
}
128129

129130
return &store.SubmitStat{
130-
StatTime: rangeStart,
131+
StatTime: *rangeStart,
131132
StatType: descStatType,
132133
FileCount: srcStat.FileCount,
133134
FileTotal: destStat.FileCount + srcStat.FileCount,

‎store/eth_data.go

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎store/store_address.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package store
22

33
import (
4+
"time"
5+
46
"github.com/Conflux-Chain/go-conflux-util/store/mysql"
57
"gorm.io/gorm"
6-
"time"
78
)
89

910
type Address struct {
@@ -26,7 +27,7 @@ func newAddressStore(db *gorm.DB) *AddressStore {
2627
}
2728
}
2829

29-
func (as *AddressStore) Add(dbTx *gorm.DB, address string, blockTime time.Time) (uint64, error) {
30+
func (as *AddressStore) Add(address string, blockTime time.Time) (uint64, error) {
3031
var addr Address
3132
existed, err := as.Store.Exists(&addr, "address = ?", address) //TODO using LRU cache for improving the query performance
3233
if err != nil {
@@ -40,10 +41,8 @@ func (as *AddressStore) Add(dbTx *gorm.DB, address string, blockTime time.Time)
4041
Address: address,
4142
BlockTime: blockTime,
4243
}
43-
if dbTx == nil {
44-
dbTx = as.Store.DB
45-
}
46-
if err := dbTx.Create(&addr).Error; err != nil {
44+
45+
if err := as.DB.Create(&addr).Error; err != nil {
4746
return 0, err
4847
}
4948

‎store/store_block.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,17 @@ package store
22

33
import (
44
"database/sql"
5+
"time"
6+
57
"github.com/Conflux-Chain/go-conflux-util/store/mysql"
68
"github.com/openweb3/web3go/types"
79
"gorm.io/gorm"
8-
"time"
910
)
1011

1112
type Block struct {
1213
BlockNumber uint64 `gorm:"primaryKey;autoIncrement:false"`
1314
Hash string `gorm:"size:66;not null"`
14-
BlockTime time.Time `gorm:"not null;index:idx_block_time,sort:desc"`
15+
BlockTime time.Time `gorm:"not null;index:idx_block_time"`
1516
}
1617

1718
func NewBlock(data *types.Block) *Block {

‎store/store_submit.go

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,36 @@ package store
22

33
import (
44
"encoding/json"
5+
"math/big"
6+
"time"
7+
58
"github.com/Conflux-Chain/go-conflux-util/store/mysql"
69
"github.com/ethereum/go-ethereum/common"
710
"github.com/openweb3/web3go/types"
811
"github.com/pkg/errors"
912
"github.com/shopspring/decimal"
1013
"github.com/zero-gravity-labs/zerog-storage-client/contract"
1114
"gorm.io/gorm"
12-
"math/big"
13-
"time"
1415
)
1516

17+
// TODO one table for overall submissions and another table for submissions of specific account.
18+
1619
type Submit struct {
1720
ID uint64 `gorm:"index:idx_sender_id,priority:2"`
18-
BlockNumber uint64 `gorm:"not null;index:idx_bn"`
19-
BlockTime time.Time `gorm:"not null;index:idx_blockTime,sort:desc"`
20-
TxHash string `gorm:"size:66;not null;index:idx_hash"`
21-
22-
SubmissionIndex uint64 `gorm:"not null"`
23-
RootHash string `gorm:"size:66;index:idx_root"`
24-
Sender string `gorm:"-"`
25-
SenderID uint64 `gorm:"not null;index:idx_sender_id,priority:1"`
26-
Length uint64 `gorm:"not null"`
27-
Finalized bool `gorm:"default:false"`
28-
Value decimal.Decimal `gorm:"type:varchar(78);not null"`
21+
BlockNumber uint64 `gorm:"not null"`
22+
BlockTime time.Time `gorm:"not null"`
23+
TxHash string `gorm:"size:66;not null"`
24+
25+
// TODO use as primary key?
26+
SubmissionIndex uint64 `gorm:"not null;index:idx_seq"`
27+
RootHash string `gorm:"size:66;index:idx_root"`
28+
Sender string `gorm:"-"`
29+
SenderID uint64 `gorm:"not null;index:idx_sender_id,priority:1"`
30+
Length uint64 `gorm:"not null"`
31+
// TODO supports more status, including L1 and L2 status
32+
Finalized bool `gorm:"default:false"`
33+
// TODO change to Fee. how about use decimal(64,18) as sql type?
34+
Value decimal.Decimal `gorm:"type:varchar(78);not null"`
2935

3036
Extra []byte `gorm:"type:mediumText"` // json field
3137
}

‎store/store_submit_stat.go

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,24 @@
11
package store
22

33
import (
4+
"time"
5+
46
"github.com/Conflux-Chain/go-conflux-util/store/mysql"
57
"github.com/pkg/errors"
68
"gorm.io/gorm"
7-
"time"
89
)
910

1011
type SubmitStat struct {
11-
ID uint64 `gorm:"primaryKey" json:"-"`
12-
StatTime *time.Time `gorm:"not null;index:idx_statTime_statType,unique,priority:1" json:"statTime"`
13-
StatType string `gorm:"type:char(3);not null;index:idx_statTime_statType,unique,priority:2" json:"-"`
14-
FileCount uint64 `gorm:"not null;default:0" json:"fileCount"` // Number of files in a specific time interval
15-
FileTotal uint64 `gorm:"not null;default:0" json:"fileTotal"` // Total number of files by a certain time
16-
DataSize uint64 `gorm:"not null;default:0" json:"dataSize"` // Size of storage data in a specific time interval
17-
DataTotal uint64 `gorm:"not null;default:0" json:"dataTotal"` // Total Size of storage data by a certain time
18-
BaseFee uint64 `gorm:"not null;default:0" json:"baseFee"` // The base fee for storage
19-
BaseFeeTotal uint64 `gorm:"not null;default:0" json:"baseFeeTotal"` // The total base fee for storage
12+
ID uint64 `json:"-"`
13+
StatType string `gorm:"size:4;not null;uniqueIndex:idx_statType_statTime,priority:1" json:"-"`
14+
StatTime time.Time `gorm:"not null;uniqueIndex:idx_statType_statTime,priority:2" json:"statTime"`
15+
FileCount uint64 `gorm:"not null;default:0" json:"fileCount"` // Number of files in a specific time interval
16+
FileTotal uint64 `gorm:"not null;default:0" json:"fileTotal"` // Total number of files by a certain time
17+
DataSize uint64 `gorm:"not null;default:0" json:"dataSize"` // Size of storage data in a specific time interval
18+
DataTotal uint64 `gorm:"not null;default:0" json:"dataTotal"` // Total Size of storage data by a certain time
19+
// TODO not enough for blockchain value of decimals 18.
20+
BaseFee uint64 `gorm:"not null;default:0" json:"baseFee"` // The base fee for storage
21+
BaseFeeTotal uint64 `gorm:"not null;default:0" json:"baseFeeTotal"` // The total base fee for storage
2022
}
2123

2224
func (SubmitStat) TableName() string {
@@ -35,7 +37,7 @@ func newSubmitStatStore(db *gorm.DB) *SubmitStatStore {
3537

3638
func (t *SubmitStatStore) LastByType(statType string) (*SubmitStat, error) {
3739
var submitStat SubmitStat
38-
err := t.Store.DB.Where("stat_type = ?", statType).Last(&submitStat).Error
40+
err := t.Store.DB.Where("stat_type = ?", statType).Order("stat_time asc").Last(&submitStat).Error
3941
if errors.Is(err, gorm.ErrRecordNotFound) {
4042
return nil, nil
4143
}
@@ -59,13 +61,13 @@ func (t *SubmitStatStore) Sum(startTime, endTime *time.Time, statType string) (*
5961
db := t.DB.Model(&SubmitStat{}).Select(`IFNULL(sum(file_count), 0) as file_count,
6062
IFNULL(sum(data_size), 0) as data_size, IFNULL(sum(base_fee), 0) as base_fee`)
6163
if startTime != nil && endTime != nil {
62-
db = db.Where("stat_time >= ? and stat_time < ? and stat_type = ?", startTime, endTime, statType)
64+
db = db.Where("stat_type = ? and stat_time >= ? and stat_time < ?", statType, startTime, endTime)
6365
}
6466
if startTime != nil && endTime == nil {
65-
db = db.Where("stat_time >= ? and stat_type = ?", startTime, statType)
67+
db = db.Where("stat_type = ? and stat_time >= ?", statType, startTime)
6668
}
6769
if startTime == nil && endTime != nil {
68-
db = db.Where("stat_time < ? and stat_type = ?", endTime, statType)
70+
db = db.Where("stat_type = ? and stat_time < ?", statType, endTime)
6971
}
7072

7173
var sum SubmitStatResult
@@ -82,5 +84,5 @@ func (t *SubmitStatStore) Add(dbTx *gorm.DB, submitStat []*SubmitStat) error {
8284
}
8385

8486
func (t *SubmitStatStore) Del(dbTx *gorm.DB, submitStat *SubmitStat) error {
85-
return dbTx.Where("stat_time = ? and stat_type = ?", submitStat.StatTime, submitStat.StatType).Delete(&SubmitStat{}).Error
87+
return dbTx.Where("stat_type = ? and stat_time = ?", submitStat.StatType, submitStat.StatTime).Delete(&SubmitStat{}).Error
8688
}

‎sync/blockchain.go

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,32 @@ package sync
22

33
import (
44
"context"
5+
56
set "github.com/deckarep/golang-set"
67
"github.com/ethereum/go-ethereum/common"
78
"github.com/openweb3/go-rpc-provider"
89
"github.com/openweb3/web3go"
910
"github.com/openweb3/web3go/types"
1011
"github.com/pkg/errors"
11-
"github.com/zero-gravity-labs/zerog-storage-scan/store"
1212
)
1313

1414
var (
1515
ErrNotFound = errors.New("not found")
1616
ErrChainReorged = errors.New("chain re-orged")
1717
)
1818

19+
type EthData struct {
20+
Number uint64
21+
Block *types.Block
22+
Receipts map[common.Hash]*types.Receipt
23+
Logs []types.Log
24+
}
25+
1926
func isTxExecutedInBlock(tx *types.TransactionDetail, receipt *types.Receipt) bool {
2027
return tx != nil && receipt.Status != nil && *receipt.Status < 2
2128
}
2229

23-
func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthData, error) {
30+
func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*EthData, error) {
2431
// get block
2532
block, err := w3c.Eth.BlockByNumber(types.BlockNumber(blockNumber), true)
2633
if err != nil {
@@ -48,13 +55,10 @@ func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthDat
4855
}
4956
for i := 0; i < len(blockTxs); i++ {
5057
tx := blockTxs[i]
51-
var receipt *types.Receipt
52-
receipt = &blockReceipts[i]
58+
receipt := &blockReceipts[i]
5359

5460
// check re-org
5561
switch {
56-
case receipt == nil: // receipt shouldn't be nil unless chain re-org
57-
return nil, errors.WithMessage(ErrChainReorged, "tx receipt nil")
5862
case receipt.BlockHash != block.Hash:
5963
return nil, errors.WithMessagef(ErrChainReorged, "receipt block hash mismatch, rcptBlkHash %v, blkHash %v",
6064
receipt.BlockHash, block.Hash)
@@ -68,10 +72,10 @@ func getEthDataByReceipts(w3c *web3go.Client, blockNumber uint64) (*store.EthDat
6872
txReceipts[tx.Hash] = receipt
6973
}
7074

71-
return &store.EthData{Number: blockNumber, Block: block, Receipts: txReceipts}, nil
75+
return &EthData{Number: blockNumber, Block: block, Receipts: txReceipts}, nil
7276
}
7377

74-
func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Address, flowSubmitSig common.Hash) (*store.EthData, error) {
78+
func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Address, flowSubmitSig common.Hash) (*EthData, error) {
7579
// get block
7680
block, err := w3c.Eth.BlockByNumber(types.BlockNumber(blockNumber), true)
7781
if err != nil {
@@ -111,7 +115,7 @@ func getEthDataByLogs(w3c *web3go.Client, blockNumber uint64, flowAddr common.Ad
111115
logs = append(logs, log)
112116
}
113117

114-
return &store.EthData{Number: blockNumber, Block: block, Logs: logs}, nil
118+
return &EthData{Number: blockNumber, Block: block, Logs: logs}, nil
115119
}
116120

117121
func batchGetFlowSubmits(w3c *web3go.Client, blockFrom, blockTo uint64, flowAddr common.Address,

‎sync/catchup.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ package sync
22

33
import (
44
"context"
5+
"math/big"
6+
"strings"
7+
"time"
8+
59
viperutil "github.com/Conflux-Chain/go-conflux-util/viper"
610
"github.com/ethereum/go-ethereum/common"
711
"github.com/openweb3/web3go"
@@ -10,9 +14,6 @@ import (
1014
"github.com/sirupsen/logrus"
1115
nhContract "github.com/zero-gravity-labs/zerog-storage-scan/contract"
1216
"github.com/zero-gravity-labs/zerog-storage-scan/store"
13-
"math/big"
14-
"strings"
15-
"time"
1617
)
1718

1819
type CatchupSyncer struct {
@@ -215,7 +216,7 @@ func (s *CatchupSyncer) convertSubmits(logs []types.Log, blockNum2TimeMap map[ui
215216
return nil, err
216217
}
217218

218-
senderId, err := s.db.AddressStore.Add(nil, submit.Sender, blockTime)
219+
senderId, err := s.db.AddressStore.Add(submit.Sender, blockTime)
219220
if err != nil {
220221
return nil, err
221222
}

0 commit comments

Comments
 (0)