Merge branch 'main' into dependabot/pip/scripts/qa/reporting/pillow-1…

…0.2.0

.github/CODEOWNERS

@@ -7,7 +7,7 @@
 # global owners are only requested if there isn't a more specific
 # codeowner specified below. For this reason, the global codeowners
 # are often repeated in package-level definitions.
-*     @evan-forbes @cmwaters
+*     @evan-forbes @cmwaters @staheri14 @rach-id @ninabarbakadze @rootulp
 
 # Overrides for tooling packages
 docs/celestia-architecture @liamsi @adlerjohn

consensus/metrics.go

@@ -22,6 +22,8 @@ const (
 type Metrics struct {
 	// Height of the chain.
 	Height metrics.Gauge
+	// The height when the metrics started from
+	StartHeight metrics.Gauge
 
 	// ValidatorLastSignedHeight of a validator.
 	ValidatorLastSignedHeight metrics.Gauge
@@ -88,6 +90,12 @@ type Metrics struct {
 	// timestamp and the timestamp of the latest prevote in a round where 100%
 	// of the voting power on the network issued prevotes.
 	FullPrevoteMessageDelay metrics.Gauge
+
+	// The amount of proposals that were rejected by the application.
+	ApplicationRejectedProposals metrics.Counter
+
+	// The amount of proposals that failed to be received in time
+	TimedOutProposals metrics.Counter
 }
 
 // PrometheusMetrics returns Metrics build using Prometheus client library.
@@ -105,6 +113,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Name:      "height",
 			Help:      "Height of the chain.",
 		}, labels).With(labelsAndValues...),
+		StartHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "start_height",
+			Help:      "Height that metrics began",
+		}, labels).With(labelsAndValues...),
 		Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
@@ -241,13 +255,26 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Help: "Difference in seconds between the proposal timestamp and the timestamp " +
 				"of the latest prevote that achieved 100% of the voting power in the prevote step.",
 		}, labels).With(labelsAndValues...),
+		ApplicationRejectedProposals: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "application_rejected_proposals",
+			Help:      "Number of proposals rejected by the application",
+		}, labels).With(labelsAndValues...),
+		TimedOutProposals: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "timed_out_proposals",
+			Help:      "Number of proposals that failed to be received in time",
+		}, labels).With(labelsAndValues...),
 	}
 }
 
 // NopMetrics returns no-op Metrics.
 func NopMetrics() *Metrics {
 	return &Metrics{
-		Height: discard.NewGauge(),
+		Height:      discard.NewGauge(),
+		StartHeight: discard.NewGauge(),
 
 		ValidatorLastSignedHeight: discard.NewGauge(),
 
@@ -265,16 +292,18 @@ func NopMetrics() *Metrics {
 
 		BlockIntervalSeconds: discard.NewHistogram(),
 
-		NumTxs:                    discard.NewGauge(),
-		BlockSizeBytes:            discard.NewGauge(),
-		TotalTxs:                  discard.NewGauge(),
-		CommittedHeight:           discard.NewGauge(),
-		FastSyncing:               discard.NewGauge(),
-		StateSyncing:              discard.NewGauge(),
-		BlockParts:                discard.NewCounter(),
-		BlockGossipPartsReceived:  discard.NewCounter(),
-		QuorumPrevoteMessageDelay: discard.NewGauge(),
-		FullPrevoteMessageDelay:   discard.NewGauge(),
+		NumTxs:                       discard.NewGauge(),
+		BlockSizeBytes:               discard.NewGauge(),
+		TotalTxs:                     discard.NewGauge(),
+		CommittedHeight:              discard.NewGauge(),
+		FastSyncing:                  discard.NewGauge(),
+		StateSyncing:                 discard.NewGauge(),
+		BlockParts:                   discard.NewCounter(),
+		BlockGossipPartsReceived:     discard.NewCounter(),
+		QuorumPrevoteMessageDelay:    discard.NewGauge(),
+		FullPrevoteMessageDelay:      discard.NewGauge(),
+		ApplicationRejectedProposals: discard.NewCounter(),
+		TimedOutProposals:            discard.NewCounter(),
 	}
 }
 

consensus/reactor.go

@@ -353,8 +353,12 @@ func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) {
 		case *VoteMessage:
 			cs := conR.conS
 			cs.mtx.RLock()
-			height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size()
+			height, round, valSize, lastCommitSize := cs.Height, cs.Round,
+				cs.Validators.Size(), cs.LastCommit.Size()
 			cs.mtx.RUnlock()
+
+			schema.WriteVote(conR.traceClient, height, round, msg.Vote, e.Src.ID(), schema.TransferTypeDownload)
+
 			ps.EnsureVoteBitArrays(height, valSize)
 			ps.EnsureVoteBitArrays(height-1, lastCommitSize)
 			ps.SetHasVote(msg.Vote)
@@ -763,7 +767,7 @@ OUTER_LOOP:
 		// Special catchup logic.
 		// If peer is lagging by height 1, send LastCommit.
 		if prs.Height != 0 && rs.Height == prs.Height+1 {
-			if ps.PickSendVote(rs.LastCommit) {
+			if conR.pickSendVoteAndTrace(rs.LastCommit, rs, ps) {
 				logger.Debug("Picked rs.LastCommit to send", "height", prs.Height)
 				continue OUTER_LOOP
 			}
@@ -776,8 +780,11 @@ OUTER_LOOP:
 			// Load the block commit for prs.Height,
 			// which contains precommit signatures for prs.Height.
 			if commit := conR.conS.blockStore.LoadBlockCommit(prs.Height); commit != nil {
-				if ps.PickSendVote(commit) {
+				vote := ps.PickSendVote(commit)
+				if vote != nil {
 					logger.Debug("Picked Catchup commit to send", "height", prs.Height)
+					schema.WriteVote(conR.traceClient, rs.Height, rs.Round, vote,
+						ps.peer.ID(), schema.TransferTypeUpload)
 					continue OUTER_LOOP
 				}
 			}
@@ -799,6 +806,18 @@ OUTER_LOOP:
 	}
 }
 
+// pickSendVoteAndTrace picks a vote to send and traces it.
+// It returns true if a vote is sent.
+// Note that it is a wrapper around PickSendVote with the addition of tracing the vote.
+func (conR *Reactor) pickSendVoteAndTrace(votes types.VoteSetReader, rs *cstypes.RoundState, ps *PeerState) bool {
+	vote := ps.PickSendVote(votes)
+	if vote != nil { // if a vote is sent, trace it
+		schema.WriteVote(conR.traceClient, rs.Height, rs.Round, vote,
+			ps.peer.ID(), schema.TransferTypeUpload)
+		return true
+	}
+	return false
+}
 func (conR *Reactor) gossipVotesForHeight(
 	logger log.Logger,
 	rs *cstypes.RoundState,
@@ -808,15 +827,15 @@ func (conR *Reactor) gossipVotesForHeight(
 
 	// If there are lastCommits to send...
 	if prs.Step == cstypes.RoundStepNewHeight {
-		if ps.PickSendVote(rs.LastCommit) {
+		if conR.pickSendVoteAndTrace(rs.LastCommit, rs, ps) {
 			logger.Debug("Picked rs.LastCommit to send")
 			return true
 		}
 	}
 	// If there are POL prevotes to send...
 	if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 {
 		if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil {
-			if ps.PickSendVote(polPrevotes) {
+			if conR.pickSendVoteAndTrace(polPrevotes, rs, ps) {
 				logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send",
 					"round", prs.ProposalPOLRound)
 				return true
@@ -825,29 +844,29 @@ func (conR *Reactor) gossipVotesForHeight(
 	}
 	// If there are prevotes to send...
 	if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round {
-		if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) {
+		if conR.pickSendVoteAndTrace(rs.Votes.Prevotes(prs.Round), rs, ps) {
 			logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
 			return true
 		}
 	}
 	// If there are precommits to send...
 	if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round {
-		if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) {
+		if conR.pickSendVoteAndTrace(rs.Votes.Precommits(prs.Round), rs, ps) {
 			logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round)
 			return true
 		}
 	}
 	// If there are prevotes to send...Needed because of validBlock mechanism
 	if prs.Round != -1 && prs.Round <= rs.Round {
-		if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) {
+		if conR.pickSendVoteAndTrace(rs.Votes.Prevotes(prs.Round), rs, ps) {
 			logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
 			return true
 		}
 	}
 	// If there are POLPrevotes to send...
 	if prs.ProposalPOLRound != -1 {
 		if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil {
-			if ps.PickSendVote(polPrevotes) {
+			if conR.pickSendVoteAndTrace(polPrevotes, rs, ps) {
 				logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send",
 					"round", prs.ProposalPOLRound)
 				return true
@@ -1163,8 +1182,8 @@ func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index in
 }
 
 // PickSendVote picks a vote and sends it to the peer.
-// Returns true if vote was sent.
-func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool {
+// Returns the vote if vote was sent. Otherwise, returns nil.
+func (ps *PeerState) PickSendVote(votes types.VoteSetReader) *types.Vote {
 	if vote, ok := ps.PickVoteToSend(votes); ok {
 		ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote)
 		if p2p.SendEnvelopeShim(ps.peer, p2p.Envelope{ //nolint: staticcheck
@@ -1174,11 +1193,11 @@ func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool {
 			},
 		}, ps.logger) {
 			ps.SetHasVote(vote)
-			return true
+			return vote
 		}
-		return false
+		return nil
 	}
-	return false
+	return nil
 }
 
 // PickVoteToSend picks a vote to send to the peer.

consensus/replay.go

@@ -252,15 +252,10 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) (string, error) {
 	}
 	appHash := res.LastBlockAppHash
 
-	h.logger.Info("ABCI Handshake App Info",
-		"height", blockHeight,
-		"hash", appHash,
-		"software-version", res.Version,
-		"protocol-version", res.AppVersion,
-	)
-
-	// Only set the version if there is no existing state.
-	if h.initialState.LastBlockHeight == 0 {
+	appVersion := h.initialState.Version.Consensus.App
+	// set app version if it's not set via genesis
+	if h.initialState.LastBlockHeight == 0 && appVersion == 0 && res.AppVersion != 0 {
+		appVersion = res.AppVersion
 		h.initialState.Version.Consensus.App = res.AppVersion
 	}
 
@@ -271,7 +266,10 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) (string, error) {
 	}
 
 	h.logger.Info("Completed ABCI Handshake - CometBFT and App are synced",
-		"appHeight", blockHeight, "appHash", appHash)
+		"appHeight", blockHeight,
+		"appHash", appHash,
+		"appVersion", appVersion,
+	)
 
 	// TODO: (on restart) replay mempool
 

consensus/state.go

@@ -315,6 +315,8 @@ func (cs *State) OnStart() error {
 		}
 	}
 
+	cs.metrics.StartHeight.Set(float64(cs.Height))
+
 	// we need the timeoutRoutine for replay so
 	// we don't block on the tick chan.
 	// NOTE: we will get a build up of garbage go routines
@@ -1281,6 +1283,7 @@ func (cs *State) defaultDoPrevote(height int64, round int32) {
 	// If ProposalBlock is nil, prevote nil.
 	if cs.ProposalBlock == nil {
 		logger.Debug("prevote step: ProposalBlock is nil")
+		cs.metrics.TimedOutProposals.Add(1)
 		cs.signAddVote(cmtproto.PrevoteType, nil, types.PartSetHeader{})
 		return
 	}
@@ -1297,12 +1300,14 @@ func (cs *State) defaultDoPrevote(height int64, round int32) {
 	stateMachineValidBlock, err := cs.blockExec.ProcessProposal(cs.ProposalBlock, cs.state)
 	if err != nil {
 		cs.Logger.Error("state machine returned an error when trying to process proposal block", "err", err)
+		return
 	}
 
 	// Vote nil if application invalidated the block
 	if !stateMachineValidBlock {
 		// The app says we must vote nil
 		logger.Error("prevote step: the application deems this block to be mustVoteNil", "err", err)
+		cs.metrics.ApplicationRejectedProposals.Add(1)
 		cs.signAddVote(cmtproto.PrevoteType, nil, types.PartSetHeader{})
 		return
 	}

docs/qa/CometBFT-QA-34.md

@@ -197,7 +197,7 @@ The thick red dashed line represents the moving average over a sliding window of
 
 #### Baseline
 
-The following graph shows the that the number of peers was stable throughout the experiment.
+The following graph shows that the number of peers was stable throughout the experiment.
 Seed nodes typically have a higher number of peers.
 The fact that non-seed nodes reach more than 50 peers is due to
 [#9548](https://github.com/cometbft/cometbft/issues/9548).
@@ -358,7 +358,7 @@ As expected, the average plot also looks similar.
 The comparison of the baseline results and the homogeneous case show that both scenarios had similar numbers and are therefore equivalent.
 
 The mixed nodes cases show that networks operate normally with a mix of compatible Tendermint Core and CometBFT versions.
-Although not the main goal, a comparison of metric numbers with the homogeneous case and the baseline scenarios show similar results and therefore we can conclude that mixing compatible Tendermint Core and CometBFT introduces not performance degradation.
+Although not the main goal, a comparison of metric numbers with the homogeneous case and the baseline scenarios show similar results and therefore we can conclude that mixing compatible Tendermint Core and CometBFT introduces no performance degradation.
 
 A conclusion of these tests is shown in the following table, along with the commit versions used in the experiments.