From 658d4ebeac138ce3ca5416baa3b034b090a58eac Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 31 May 2019 13:53:15 +0800 Subject: [PATCH 1/7] delete db table script which created by explorer --- script/mongodb.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/script/mongodb.js b/script/mongodb.js index 3fb8387..681a028 100644 --- a/script/mongodb.js +++ b/script/mongodb.js @@ -15,13 +15,9 @@ db.createCollection("sync_task"); db.createCollection("tx_common"); db.createCollection("proposal"); db.createCollection("tx_msg"); -db.createCollection("power_change");//explorer -db.createCollection("uptime_change"); db.createCollection("sync_conf"); db.createCollection("mgo_txn"); db.createCollection("mgo_txn.stash"); -db.createCollection("ex_tx_num_stat"); - // create index @@ -42,12 +38,8 @@ db.tx_common.createIndex({"to": 1}); db.tx_common.createIndex({"type": 1}); db.tx_common.createIndex({"status": 1}); -db.power_change.createIndex({"height": 1, "address": 1}, {"unique": true}); - - db.proposal.createIndex({"proposal_id": 1}, {"unique": true}); db.tx_msg.createIndex({"hash": 1}, {"unique": true}); -db.ex_tx_num_stat.createIndex({"date": -1}, {"unique": true}); // init data db.sync_conf.insert({"block_num_per_worker_handle": 50, "max_worker_sleep_time": 120}); @@ -55,7 +47,6 @@ db.sync_conf.insert({"block_num_per_worker_handle": 50, "max_worker_sleep_time": // drop collection // db.account.drop(); // db.block.drop(); -// db.power_change.drop(); // db.proposal.drop(); // db.stake_role_candidate.drop(); // db.sync_task.drop(); From f8d733af5a15f6ecea4373bfd89cd2ad2f534028 Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 31 May 2019 17:04:39 +0800 Subject: [PATCH 2/7] parse and save block proposal info --- service/handler/block.go | 9 +++++---- service/handler/block_test.go | 23 +++++++++++++++++++++++ store/document/block.go | 17 +++++++++-------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/service/handler/block.go b/service/handler/block.go index e37e448..0a0018e 100644 --- a/service/handler/block.go +++ b/service/handler/block.go @@ -36,10 +36,11 @@ func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.V } docBlock := document.Block{ - Height: meta.Header.Height, - Hash: hexFunc(meta.BlockID.Hash), - Time: meta.Header.Time, - NumTxs: meta.Header.NumTxs, + Height: meta.Header.Height, + Hash: hexFunc(meta.BlockID.Hash), + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + ProposalAddress: block.Header.ProposerAddress.String(), } lastBlockId := document.BlockID{ diff --git a/service/handler/block_test.go b/service/handler/block_test.go index d26857b..cd80806 100644 --- a/service/handler/block_test.go +++ b/service/handler/block_test.go @@ -48,3 +48,26 @@ func TestParseBlockResult(t *testing.T) { bz, _ := json.Marshal(v) fmt.Println(string(bz)) } + +func TestParseBlock(t *testing.T) { + blockHeight := int64(88) + + client := helper.GetClient() + defer client.Release() + + if res, err := client.Block(&blockHeight); err != nil { + t.Fatal(err) + } else { + var validators []*types.Validator + valRes, err := client.Validators(&blockHeight) + if err != nil { + t.Error(err) + } else { + validators = valRes.Validators + } + blockDoc := ParseBlock(res.BlockMeta, res.Block, validators, nil) + + resBytes, _ := json.MarshalIndent(blockDoc, "", "\t") + t.Log(string(resBytes)) + } +} diff --git a/store/document/block.go b/store/document/block.go index bd11e8f..54d1d52 100644 --- a/store/document/block.go +++ b/store/document/block.go @@ -14,14 +14,15 @@ const ( ) type Block struct { - Height int64 `bson:"height"` - Hash string `bson:"hash"` - Time time.Time `bson:"time"` - NumTxs int64 `bson:"num_txs"` - Meta BlockMeta `bson:"meta"` - Block BlockContent `bson:"block"` - Validators []Validator `bson:"validators"` - Result BlockResults `bson:"results"` + Height int64 `bson:"height"` + Hash string `bson:"hash"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + ProposalAddress string `bson:"proposal_address"` + Meta BlockMeta `bson:"meta"` + Block BlockContent `bson:"block"` + Validators []Validator `bson:"validators"` + Result BlockResults `bson:"results"` } type BlockMeta struct { From 3c7b91aa15148fded46b12df2cc8276fc95b18c6 Mon Sep 17 00:00:00 2001 From: kaifei Date: Tue, 11 Jun 2019 19:02:59 +0800 Subject: [PATCH 3/7] save validator commission rate while parse editValidator tx --- conf/server/types.go | 9 +-------- store/document/tx.go | 3 ++- util/helper/tx.go | 7 +++++++ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf/server/types.go b/conf/server/types.go index 193a9e6..c90d80e 100644 --- a/conf/server/types.go +++ b/conf/server/types.go @@ -11,7 +11,6 @@ import ( var ( BlockChainMonitorUrl = []string{"tcp://35.201.147.145:30657"} - ChainId = "fuxi" WorkerNumCreateTask = 2 WorkerNumExecuteTask = 60 @@ -32,12 +31,6 @@ func init() { logger.Info("Env Value", logger.Any(constant.EnvNameSerNetworkFullNode, BlockChainMonitorUrl)) - chainId, found := os.LookupEnv(constant.EnvNameSerNetworkChainId) - if found { - ChainId = chainId - } - logger.Info("Env Value", logger.String(constant.EnvNameSerNetworkChainId, ChainId)) - workerNumCreateTask, found := os.LookupEnv(constant.EnvNameWorkerNumCreateTask) if found { var err error @@ -62,5 +55,5 @@ func init() { if found { Network = network } - logger.Info("Env Value", logger.String(constant.EnvNameNetwork, network)) + logger.Info("Env Value", logger.String(constant.EnvNameNetwork, Network)) } diff --git a/store/document/tx.go b/store/document/tx.go index 5d26320..5156b88 100644 --- a/store/document/tx.go +++ b/store/document/tx.go @@ -56,7 +56,8 @@ type StakeCreateValidator struct { } type StakeEditValidator struct { - Description ValDescription `bson:"description"` + CommissionRate string `bson:"commission_rate"` + Description ValDescription `bson:"description"` } type Signer struct { diff --git a/util/helper/tx.go b/util/helper/tx.go index c5e604e..22c3746 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -152,9 +152,16 @@ func ParseTx(txBytes itypes.Tx, block *itypes.Block) document.CommonTx { Website: msg.Website, Details: msg.Details, } + docTx.StakeEditValidator = document.StakeEditValidator{ Description: valDes, } + commissionRate := msg.CommissionRate + if commissionRate == nil { + docTx.StakeEditValidator.CommissionRate = "" + } else { + docTx.StakeEditValidator.CommissionRate = commissionRate.String() + } return docTx case itypes.MsgStakeDelegate: From df651c7e347930c6d96f8025501983bd1a4b69f8 Mon Sep 17 00:00:00 2001 From: kaifei Date: Wed, 19 Jun 2019 10:47:52 +0800 Subject: [PATCH 4/7] no longer to save validators info in every block --- service/handler/validator.go | 138 ------------------------------ service/handler/validator_test.go | 61 ------------- service/task/task_execute.go | 6 -- 3 files changed, 205 deletions(-) delete mode 100644 service/handler/validator.go delete mode 100644 service/handler/validator_test.go diff --git a/service/handler/validator.go b/service/handler/validator.go deleted file mode 100644 index 32d75b7..0000000 --- a/service/handler/validator.go +++ /dev/null @@ -1,138 +0,0 @@ -package handler - -import ( - "github.com/irisnet/irishub-sync/logger" - "github.com/irisnet/irishub-sync/store/document" - "github.com/irisnet/irishub-sync/types" - "github.com/irisnet/irishub-sync/util/helper" -) - -// compare validatorSet stored in irishub and validatorSet stored in db -// if dbCandidates not equal chainValidators, execute two step as follow. -// first, remove all validators stored in db -// second, store latest validators which query from sdk store into db -// note: this function isn't thread safe, should be invoked during watch block -// not fast sync -func CompareAndUpdateValidators() { - var ( - methodName = "CompareAndUpdateValidators" - - candidateModel document.Candidate - ) - - // get all validatorSets from db - dbCandidates := candidateModel.QueryAll() - - // get all validatorSets from blockChain - validators := helper.GetValidators() - - logger.Debug("Get Validators from blockchain", logger.Any("Validators", validators)) - var chainValidators []document.Candidate - for _, validator := range validators { - // build validator document struct by stake.validator - doc := BuildValidatorDocument(validator) - chainValidators = append(chainValidators, doc) - } - - if len(chainValidators) == 0 { - logger.Error("Validators is empty,Update Validators Failed") - return - } - - // dbCandidates not equal chainValidators - if compareValidators(dbCandidates, chainValidators) { - // remove all data which stored in db - if err := candidateModel.RemoveCandidates(); err != nil { - logger.Error("RemoveCandidates err ", logger.String("method", methodName), logger.String("err", err.Error())) - } - - // store latest validators into db - if err := candidateModel.SaveAll(chainValidators); err != nil { - logger.Error("SaveAll", logger.String("method", methodName), logger.String("err", err.Error())) - } - } -} - -func BuildValidatorDocument(v types.StakeValidator) document.Candidate { - description := document.ValDescription{ - Moniker: v.Description.Moniker, - Identity: v.Description.Identity, - Website: v.Description.Website, - Details: v.Description.Details, - } - - floatTokens := helper.ParseFloat(v.Tokens.String()) - floatDelegatorShares := helper.ParseFloat(v.DelegatorShares.String()) - pubKey, err := types.Bech32ifyValPub(v.ConsPubKey) - if err != nil { - logger.Error("Can't get validator pubKey", logger.String("pubKey", pubKey), logger.String("err", err.Error())) - } - doc := document.Candidate{ - Address: v.OperatorAddr.String(), - PubKey: pubKey, - PubKeyAddr: v.ConsPubKey.Address().String(), - Jailed: v.Jailed, - Tokens: floatTokens, - OriginalTokens: helper.RoundString(v.Tokens.String(), 0), - DelegatorShares: floatDelegatorShares, - Description: description, - BondHeight: v.BondHeight, - Status: types.BondStatusToString(v.Status), - } - - doc.VotingPower = doc.Tokens - - return doc -} - -func compareValidators(dbVals []document.Candidate, chainVals []document.Candidate) bool { - //Candidate数量不一致 - if len(dbVals) != len(chainVals) { - logger.Info("Candidate's member amount has changed", logger.Int("db", len(dbVals)), logger.Int("blockchain", len(chainVals))) - return true - } - - chainValsMap := make(map[string]document.Candidate) - for _, v := range chainVals { - chainValsMap[v.PubKeyAddr] = v - } - - for _, v := range dbVals { - v1, ok := chainValsMap[v.PubKeyAddr] - if !ok { - logger.Info("Candidate's member has changed,removed", - logger.String("dbValue", v.PubKeyAddr), - ) - return true - } - - if v.Tokens != v1.Tokens { - logger.Info("Candidate's votingPower has changed", - logger.String("validator", v.Address), - logger.Float64("dbTokens", v1.Tokens), - logger.Float64("tmTokens", v1.Tokens), - ) - return true - } - - if v.Jailed != v1.Jailed { - logger.Info("Candidate's jailed status has changed", - logger.String("validator", v.Address), - logger.Bool("dbJailed", v.Jailed), - logger.Bool("tmJailed", v1.Jailed), - ) - return true - } - - if v.Status != v1.Status { - logger.Info("Candidate's status has changed", - logger.String("validator", v.Address), - logger.String("dbStatus", v.Status), - logger.String("tmStatus", v1.Status), - ) - return true - } - } - logger.Info("Validators Set is not changed ") - return false -} diff --git a/service/handler/validator_test.go b/service/handler/validator_test.go deleted file mode 100644 index c5c0fef..0000000 --- a/service/handler/validator_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package handler - -import ( - "fmt" - "github.com/irisnet/irishub-sync/types" - "github.com/irisnet/irishub-sync/util/helper" - "sort" - "testing" -) - -func TestCompareAndUpdateValidators(t *testing.T) { - c := helper.GetClient() - defer c.Release() - - status, _ := c.Client.Status() - res, _ := c.Client.Validators(&status.SyncInfo.LatestBlockHeight) - tmVals := res.Validators - - type args struct { - tmVals []*types.Validator - } - tests := []struct { - name string - args args - }{ - { - name: "test compare and update validators", - args: args{ - tmVals: tmVals, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - CompareAndUpdateValidators() - }) - } -} - -type Person struct { - Name string - Age int -} - -func (p Person) String() string { - return fmt.Sprintf("%s: %d", p.Name, p.Age) -} - -func TestSort(t *testing.T) { - people := []Person{ - {"Bob", 31}, - {"John", 42}, - {"Michael", 17}, - {"Jenny", 26}, - } - - sort.SliceStable(people, func(i, j int) bool { - return people[i].Age > people[j].Age - }) - fmt.Println(people) -} diff --git a/service/task/task_execute.go b/service/task/task_execute.go index 8e48a02..4ffadff 100644 --- a/service/task/task_execute.go +++ b/service/task/task_execute.go @@ -231,12 +231,6 @@ func executeTask(blockNumPerWorkerHandle, maxWorkerSleepTime int64, chanLimit ch log.Error("save docs fail", logger.String("err", err.Error())) } else { task.CurrentHeight = inProcessBlock - - if taskType == document.SyncTaskTypeFollow { - // TODO: whether can remove compareAndUpdateValidators in sync logic - // compare and update validators - handler.CompareAndUpdateValidators() - } } } else { log.Info("task worker changed", logger.Any("task_id", task.ID), From ad9ab54324e6ff714db772627ca17b589426f2cc Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 21 Jun 2019 11:09:11 +0800 Subject: [PATCH 5/7] make health check goroutine can exit correct --- script/mongodb.js | 6 ------ service/task/task_execute.go | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/script/mongodb.js b/script/mongodb.js index 082af4c..e73ff6e 100644 --- a/script/mongodb.js +++ b/script/mongodb.js @@ -10,7 +10,6 @@ // create collections db.createCollection("block"); -db.createCollection("stake_role_candidate"); db.createCollection("sync_task"); db.createCollection("tx_common"); db.createCollection("proposal"); @@ -24,9 +23,6 @@ db.createCollection("mgo_txn.stash"); db.account.createIndex({"address": 1}, {"unique": true}); db.block.createIndex({"height": -1}, {"unique": true}); -db.stake_role_candidate.createIndex({"address": 1}, {"unique": true}); -db.stake_role_candidate.createIndex({"pub_key": 1}); - db.sync_task.createIndex({"start_height": 1, "end_height": 1}, {"unique": true}); db.sync_task.createIndex({"status": 1}, {"background": true}); @@ -52,7 +48,6 @@ db.sync_conf.insert({"block_num_per_worker_handle": 50, "max_worker_sleep_time": // db.account.drop(); // db.block.drop(); // db.proposal.drop(); -// db.stake_role_candidate.drop(); // db.sync_task.drop(); // db.tx_common.drop(); // db.tx_msg.drop(); @@ -63,7 +58,6 @@ db.sync_conf.insert({"block_num_per_worker_handle": 50, "max_worker_sleep_time": // db.account.remove({}); // db.block.remove({}); // db.proposal.remove({}); -// db.stake_role_candidate.remove({}); // db.sync_task.remove({}); // db.tx_common.remove({}); // db.tx_msg.remove({}); diff --git a/service/task/task_execute.go b/service/task/task_execute.go index 4ffadff..382490b 100644 --- a/service/task/task_execute.go +++ b/service/task/task_execute.go @@ -146,7 +146,7 @@ func executeTask(blockNumPerWorkerHandle, maxWorkerSleepTime int64, chanLimit ch logger.String("task_current_worker", task.WorkerId)) } else { log.Info("task is invalid, exit health check", logger.String("task_id", taskId.Hex())) - break + return } } else { log.Error("get block chain latest height fail", logger.String("err", err.Error())) @@ -155,7 +155,7 @@ func executeTask(blockNumPerWorkerHandle, maxWorkerSleepTime int64, chanLimit ch if err == mgo.ErrNotFound { log.Info("task may be task over by other goroutine, exit health check", logger.String("task_id", taskId.Hex()), logger.String("current_worker", workerId)) - break + return } else { log.Error("get task by id and worker fail", logger.String("task_id", taskId.Hex()), logger.String("current_worker", workerId)) From 665f588e9e2aac4f8c2dbeaf8bed51382e0d389c Mon Sep 17 00:00:00 2001 From: huangweichang Date: Fri, 21 Jun 2019 19:35:11 +0800 Subject: [PATCH 6/7] save the voting end height to proposal table. [taskID: http://jira.bianjie.ai/browse/IRISSYNCS-104] --- service/handler/block.go | 11 +++++++++++ service/handler/proposal.go | 15 +++++++++++++++ service/handler/proposal_test.go | 4 ++-- store/document/proposal.go | 1 + util/constant/types.go | 1 + 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/service/handler/block.go b/service/handler/block.go index 0a0018e..11dcc05 100644 --- a/service/handler/block.go +++ b/service/handler/block.go @@ -8,6 +8,7 @@ import ( itypes "github.com/irisnet/irishub-sync/types" "github.com/irisnet/irishub-sync/util/helper" "strings" + "github.com/irisnet/irishub-sync/store" ) var ( @@ -128,6 +129,16 @@ func ParseBlock(meta *types.BlockMeta, block *types.Block, validators []*types.V docBlock.Validators = vals docBlock.Result = parseBlockResult(docBlock.Height) + if proposalId,ok := IsContainVotingEndTag(docBlock.Result.EndBlock);ok { + if proposal,err := document.QueryProposal(proposalId);err == nil { + proposal.VotingEndHeight = docBlock.Height + store.SaveOrUpdate(proposal) + }else{ + logger.Error("QueryProposal fail", logger.Int64("block", docBlock.Height), + logger.String("err", err.Error())) + } + } + // save or update account balance info and unbonding delegation info by parse block coin flow accsBalanceNeedUpdated, accsUnbondingDelegationNeedUpdated := getAccountsFromCoinFlow( docBlock.Result.EndBlock.Tags, docBlock.Height) diff --git a/service/handler/proposal.go b/service/handler/proposal.go index e68561a..c9555d7 100644 --- a/service/handler/proposal.go +++ b/service/handler/proposal.go @@ -6,6 +6,7 @@ import ( "github.com/irisnet/irishub-sync/types" "github.com/irisnet/irishub-sync/util/constant" "github.com/irisnet/irishub-sync/util/helper" + "strconv" ) func handleProposal(docTx document.CommonTx) { @@ -72,3 +73,17 @@ func isContainVotingPeriodStartTag(docTx document.CommonTx) bool { return false } + +func IsContainVotingEndTag(blockresult document.ResponseEndBlock) (uint64, bool) { + tags := blockresult.Tags + if len(tags) > 0 { + for _, tag := range tags { + if tag.Key == constant.TxTagProposalId { + proposalid,_ := strconv.ParseUint(tag.Value,10,64) + return proposalid, true + } + } + } + + return 0, false +} diff --git a/service/handler/proposal_test.go b/service/handler/proposal_test.go index 990de4d..5498316 100644 --- a/service/handler/proposal_test.go +++ b/service/handler/proposal_test.go @@ -11,7 +11,7 @@ import ( ) func TestIsContainVotingPeriodStartTag(t *testing.T) { - txHash := "A837138C2A569B7884AA94C27CC4AB791C04F1B8DD93EFC3D5BFCF3D7EB0F2F3" + txHash := "37A0127A87AA68BFE73D03C2B9A2A6A3D8E51DF242D86C845DB2D158B1617502" var tx document.CommonTx fn := func(c *mgo.Collection) error { @@ -28,7 +28,7 @@ func TestIsContainVotingPeriodStartTag(t *testing.T) { } func TestHandleProposal(t *testing.T) { - txHash := "5875062EE8B8656CF943C42983F382B5341B1B0C530062D266BD8283CA9658B0" + txHash := "37A0127A87AA68BFE73D03C2B9A2A6A3D8E51DF242D86C845DB2D158B1617502" var tx document.CommonTx fn := func(c *mgo.Collection) error { diff --git a/store/document/proposal.go b/store/document/proposal.go index 4a2ae89..2855d0b 100644 --- a/store/document/proposal.go +++ b/store/document/proposal.go @@ -25,6 +25,7 @@ type Proposal struct { VotingStartTime time.Time `bson:"voting_start_time"` VotingEndTime time.Time `bson:"voting_end_time"` VotingPeriodStartHeight int64 `bson:"voting_start_height"` + VotingEndHeight int64 `bson:"voting_end_height"` TotalDeposit store.Coins `bson:"total_deposit"` Votes []PVote `bson:"votes"` TallyResult PTallyResult `bson:"tally_result"` diff --git a/util/constant/types.go b/util/constant/types.go index b603cfa..2af88f2 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -24,6 +24,7 @@ const ( TxMsgTypeSubmitTaxUsageProposal = "SubmitTaxUsageProposal" TxTagVotingPeriodStart = "voting-period-start" + TxTagProposalId = "proposal-id" EnvNameDbAddr = "DB_ADDR" EnvNameDbUser = "DB_USER" From 2be43bb36f0608da7f58b8c92a8a4fcaf0dc8986 Mon Sep 17 00:00:00 2001 From: huangweichang Date: Mon, 24 Jun 2019 09:43:02 +0800 Subject: [PATCH 7/7] rename the TxTagProposalId to BlockTagProposalId --- service/handler/proposal.go | 2 +- util/constant/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service/handler/proposal.go b/service/handler/proposal.go index c9555d7..1fc5576 100644 --- a/service/handler/proposal.go +++ b/service/handler/proposal.go @@ -78,7 +78,7 @@ func IsContainVotingEndTag(blockresult document.ResponseEndBlock) (uint64, bool) tags := blockresult.Tags if len(tags) > 0 { for _, tag := range tags { - if tag.Key == constant.TxTagProposalId { + if tag.Key == constant.BlockTagProposalId { proposalid,_ := strconv.ParseUint(tag.Value,10,64) return proposalid, true } diff --git a/util/constant/types.go b/util/constant/types.go index 2af88f2..0c7b61d 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -24,7 +24,7 @@ const ( TxMsgTypeSubmitTaxUsageProposal = "SubmitTaxUsageProposal" TxTagVotingPeriodStart = "voting-period-start" - TxTagProposalId = "proposal-id" + BlockTagProposalId = "proposal-id" EnvNameDbAddr = "DB_ADDR" EnvNameDbUser = "DB_USER"