diff --git a/app/app.go b/app/app.go index e5db83c39..693309a9c 100644 --- a/app/app.go +++ b/app/app.go @@ -47,7 +47,8 @@ import ( "github.com/obolnetwork/charon/core/aggsigdb" "github.com/obolnetwork/charon/core/bcast" "github.com/obolnetwork/charon/core/consensus" - pbv1 "github.com/obolnetwork/charon/core/corepb/v1" + "github.com/obolnetwork/charon/core/consensus/protocols" + "github.com/obolnetwork/charon/core/consensus/qbft" "github.com/obolnetwork/charon/core/dutydb" "github.com/obolnetwork/charon/core/fetcher" "github.com/obolnetwork/charon/core/infosync" @@ -92,6 +93,7 @@ type Config struct { SimnetBMockFuzz bool TestnetConfig eth2util.Network ProcDirectory string + ConsensusProtocol string TestConfig TestConfig } @@ -257,8 +259,6 @@ func Run(ctx context.Context, conf Config) (err error) { wirePeerInfo(life, tcpNode, peerIDs, cluster.GetInitialMutationHash(), sender, conf.BuilderAPI) - consensusDebugger := consensus.NewDebugger() - // seenPubkeys channel to send seen public keys from validatorapi to monitoringapi. seenPubkeys := make(chan core.PubKey) seenPubkeysFunc := func(pk core.PubKey) { @@ -281,6 +281,8 @@ func Run(ctx context.Context, conf Config) (err error) { return err } + consensusDebugger := consensus.NewDebugger() + wireMonitoringAPI(ctx, life, conf.MonitoringAddr, conf.DebugAddr, tcpNode, eth2Cl, peerIDs, promRegistry, consensusDebugger, pubkeys, seenPubkeys, vapiCalls, len(cluster.GetValidators())) @@ -524,16 +526,25 @@ func wireCoreWorkflow(ctx context.Context, life *lifecycle.Manager, conf Config, return err } - retryer := retry.New[core.Duty](deadlineFunc) + retryer := retry.New(deadlineFunc) - cons, startCons, err := newConsensus(cluster, tcpNode, p2pKey, sender, - deadlinerFunc("consensus"), gaterFunc, consensusDebugger.AddInstance) + // Consensus + consensusController, err := consensus.NewConsensusController( + ctx, tcpNode, sender, peers, p2pKey, + deadlineFunc, gaterFunc, consensusDebugger) if err != nil { return err } + defaultConsensus := consensusController.DefaultConsensus() + startConsensusCtrl := lifecycle.HookFuncCtx(consensusController.Start) + + coreConsensus := consensusController.CurrentConsensus() // initially points to DefaultConsensus() + + // Priority protocol always uses QBFTv2. err = wirePrioritise(ctx, conf, life, tcpNode, peerIDs, int(cluster.GetThreshold()), - sender.SendReceive, cons, sched, p2pKey, deadlineFunc) + sender.SendReceive, defaultConsensus, sched, p2pKey, deadlineFunc, + consensusController, cluster.GetConsensusProtocol()) if err != nil { return err } @@ -553,12 +564,13 @@ func wireCoreWorkflow(ctx context.Context, life *lifecycle.Manager, conf Config, return err } + // Core always uses the "current" consensus that is changed dynamically. opts := []core.WireOption{ core.WithTracing(), core.WithTracking(track, inclusion), core.WithAsyncRetry(retryer), } - core.Wire(sched, fetch, cons, dutyDB, vapi, parSigDB, parSigEx, sigAgg, aggSigDB, broadcaster, opts...) + core.Wire(sched, fetch, coreConsensus, dutyDB, vapi, parSigDB, parSigEx, sigAgg, aggSigDB, broadcaster, opts...) err = wireValidatorMock(ctx, conf, eth2Cl, pubshares, sched) if err != nil { @@ -570,7 +582,7 @@ func wireCoreWorkflow(ctx context.Context, life *lifecycle.Manager, conf Config, } life.RegisterStart(lifecycle.AsyncBackground, lifecycle.StartScheduler, lifecycle.HookFuncErr(sched.Run)) - life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartP2PConsensus, startCons) + life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartP2PConsensus, startConsensusCtrl) life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartAggSigDB, lifecycle.HookFuncCtx(aggSigDB.Run)) life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartParSigDB, lifecycle.HookFuncCtx(parSigDB.Trim)) life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartTracker, lifecycle.HookFuncCtx(inclusion.Run)) @@ -585,8 +597,9 @@ func wireCoreWorkflow(ctx context.Context, life *lifecycle.Manager, conf Config, func wirePrioritise(ctx context.Context, conf Config, life *lifecycle.Manager, tcpNode host.Host, peers []peer.ID, threshold int, sendFunc p2p.SendReceiveFunc, coreCons core.Consensus, sched core.Scheduler, p2pKey *k1.PrivateKey, deadlineFunc func(duty core.Duty) (time.Time, bool), + consensusController core.ConsensusController, clusterPreferredProtocol string, ) error { - cons, ok := coreCons.(*consensus.Component) + cons, ok := coreCons.(*qbft.Consensus) if !ok { // Priority protocol not supported for leader cast. return nil @@ -602,9 +615,22 @@ func wirePrioritise(ctx context.Context, conf Config, life *lifecycle.Manager, t return err } + // The initial protocols order as defined by implementation is altered by: + // 1. Prioritizing the cluster (lock) preferred protocol to the top. + // 2. Prioritizing the protocol specified by CLI flag (cluster run) to the top. + // In all cases this prioritizes all versions of the protocol identified by name. + // The order of all these operations are important. + allProtocols := Protocols() + if clusterPreferredProtocol != "" { + allProtocols = protocols.PrioritizeProtocolsByName(clusterPreferredProtocol, allProtocols) + } + if conf.ConsensusProtocol != "" { + allProtocols = protocols.PrioritizeProtocolsByName(conf.ConsensusProtocol, allProtocols) + } + isync := infosync.New(prio, version.Supported(), - Protocols(), + allProtocols, ProposalTypes(conf.BuilderAPI, conf.SyntheticBlockProposals), ) @@ -621,6 +647,26 @@ func wirePrioritise(ctx context.Context, conf Config, life *lifecycle.Manager, t prio.Subscribe(conf.TestConfig.PrioritiseCallback) } + prio.Subscribe(func(ctx context.Context, _ core.Duty, tr []priority.TopicResult) error { + for _, t := range tr { + if t.Topic == infosync.TopicProtocol { + allProtocols := t.PrioritiesOnly() + preferredConsensusProtocol := protocols.MostPreferredConsensusProtocol(allProtocols) + preferredConsensusProtocolID := protocol.ID(preferredConsensusProtocol) + + if err := consensusController.SetCurrentConsensusForProtocol(ctx, preferredConsensusProtocolID); err != nil { + log.Error(ctx, "Failed to set current consensus protocol", err, z.Str("protocol", preferredConsensusProtocol)) + } else { + log.Info(ctx, "Current consensus protocol changed", z.Str("protocol", preferredConsensusProtocol)) + } + + break + } + } + + return nil + }) + life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartPeerInfo, lifecycle.HookFuncCtx(prio.Start)) return nil @@ -918,24 +964,6 @@ func configureEth2Client(ctx context.Context, forkVersion []byte, addrs []string return eth2Cl, nil } -// newConsensus returns a new consensus component and its start lifecycle hook. -func newConsensus(cluster *manifestpb.Cluster, tcpNode host.Host, p2pKey *k1.PrivateKey, - sender *p2p.Sender, deadliner core.Deadliner, gaterFunc core.DutyGaterFunc, - qbftSniffer func(*pbv1.SniffedConsensusInstance), -) (core.Consensus, lifecycle.IHookFunc, error) { - peers, err := manifest.ClusterPeers(cluster) - if err != nil { - return nil, nil, err - } - - comp, err := consensus.New(tcpNode, sender, peers, p2pKey, deadliner, gaterFunc, qbftSniffer) - if err != nil { - return nil, nil, err - } - - return comp, lifecycle.HookFuncCtx(comp.Start), nil -} - // createMockValidators creates mock validators identified by their public shares. func createMockValidators(pubkeys []eth2p0.BLSPubKey) beaconmock.ValidatorSet { resp := make(beaconmock.ValidatorSet) @@ -1079,7 +1107,7 @@ func (h httpServeHook) Call(context.Context) error { // Protocols returns the list of supported Protocols in order of precedence. func Protocols() []protocol.ID { var resp []protocol.ID - resp = append(resp, consensus.Protocols()...) + resp = append(resp, protocols.Protocols()...) resp = append(resp, parsigex.Protocols()...) resp = append(resp, peerinfo.Protocols()...) resp = append(resp, priority.Protocols()...) diff --git a/cluster/manifest/mutationlegacylock.go b/cluster/manifest/mutationlegacylock.go index ab97b357b..21b25f025 100644 --- a/cluster/manifest/mutationlegacylock.go +++ b/cluster/manifest/mutationlegacylock.go @@ -145,12 +145,13 @@ func transformLegacyLock(input *manifestpb.Cluster, signed *manifestpb.SignedMut } return &manifestpb.Cluster{ - Name: lock.Name, - Threshold: int32(lock.Threshold), - DkgAlgorithm: lock.DKGAlgorithm, - ForkVersion: lock.ForkVersion, - Validators: vals, - Operators: ops, + Name: lock.Name, + Threshold: int32(lock.Threshold), + DkgAlgorithm: lock.DKGAlgorithm, + ForkVersion: lock.ForkVersion, + ConsensusProtocol: lock.ConsensusProtocol, + Validators: vals, + Operators: ops, }, nil } diff --git a/cluster/manifestpb/v1/manifest.pb.go b/cluster/manifestpb/v1/manifest.pb.go index cec646a99..bc755036f 100644 --- a/cluster/manifestpb/v1/manifest.pb.go +++ b/cluster/manifestpb/v1/manifest.pb.go @@ -35,6 +35,7 @@ type Cluster struct { ForkVersion []byte `protobuf:"bytes,6,opt,name=fork_version,json=forkVersion,proto3" json:"fork_version,omitempty"` // ForkVersion is the fork version (network/chain) of the cluster. It must be 4 bytes. Operators []*Operator `protobuf:"bytes,7,rep,name=operators,proto3" json:"operators,omitempty"` // Operators is the list of operators of the cluster. Validators []*Validator `protobuf:"bytes,8,rep,name=validators,proto3" json:"validators,omitempty"` // Validators is the list of validators of the cluster. + ConsensusProtocol string `protobuf:"bytes,9,opt,name=consensus_protocol,json=consensusProtocol,proto3" json:"consensus_protocol,omitempty"` // ConsensusProtocol is the consensus protocol name preferred by the cluster, e.g. "abft". } func (x *Cluster) Reset() { @@ -123,6 +124,13 @@ func (x *Cluster) GetValidators() []*Validator { return nil } +func (x *Cluster) GetConsensusProtocol() string { + if x != nil { + return x.ConsensusProtocol + } + return "" +} + // Mutation mutates the cluster manifest. type Mutation struct { state protoimpl.MessageState @@ -562,7 +570,7 @@ var file_cluster_manifestpb_v1_manifest_proto_rawDesc = []byte{ 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x15, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, - 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xea, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, + 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x99, 0x03, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4d, 0x75, 0x74, 0x61, @@ -585,57 +593,60 @@ var file_cluster_manifestpb_v1_manifest_proto_rawDesc = []byte{ 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x6f, 0x72, 0x73, 0x22, 0x60, 0x0a, 0x08, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x28, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, - 0x79, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, - 0x65, 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3b, 0x0a, 0x08, 0x6d, 0x75, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, - 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6d, - 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x65, - 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, - 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x59, 0x0a, - 0x12, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x09, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x65, 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6d, - 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x36, 0x0a, 0x08, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x10, - 0x0a, 0x03, 0x65, 0x6e, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x6e, 0x72, - 0x22, 0xe8, 0x01, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, - 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, - 0x0a, 0x70, 0x75, 0x62, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x53, 0x68, 0x61, 0x72, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, - 0x66, 0x65, 0x65, 0x5f, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x66, 0x65, 0x65, - 0x52, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x12, 0x2d, 0x0a, 0x12, 0x77, 0x69, 0x74, 0x68, 0x64, 0x72, 0x61, 0x77, 0x61, 0x6c, 0x5f, 0x61, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x69, - 0x74, 0x68, 0x64, 0x72, 0x61, 0x77, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x3a, 0x0a, 0x19, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x17, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x73, 0x6f, 0x6e, 0x22, 0x51, 0x0a, 0x0d, 0x56, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x0a, - 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, - 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, - 0x6f, 0x72, 0x52, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x22, 0x20, - 0x0a, 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, - 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6a, 0x73, 0x6f, 0x6e, - 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x62, 0x6f, 0x6c, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x2f, 0x63, 0x68, 0x61, 0x72, 0x6f, 0x6e, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x2f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2f, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, + 0x75, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x60, 0x0a, 0x08, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x28, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x65, + 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3b, 0x0a, 0x08, 0x6d, 0x75, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, + 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6d, 0x75, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x1c, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x59, 0x0a, 0x12, + 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x43, 0x0a, 0x09, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, + 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x65, 0x64, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6d, 0x75, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x36, 0x0a, 0x08, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x10, 0x0a, + 0x03, 0x65, 0x6e, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x6e, 0x72, 0x22, + 0xe8, 0x01, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x70, 0x75, 0x62, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x09, 0x70, 0x75, 0x62, 0x53, 0x68, 0x61, 0x72, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x66, + 0x65, 0x65, 0x5f, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x66, 0x65, 0x65, 0x52, + 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x2d, 0x0a, 0x12, 0x77, 0x69, 0x74, 0x68, 0x64, 0x72, 0x61, 0x77, 0x61, 0x6c, 0x5f, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x69, 0x74, + 0x68, 0x64, 0x72, 0x61, 0x77, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x3a, + 0x0a, 0x19, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x17, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x73, 0x6f, 0x6e, 0x22, 0x51, 0x0a, 0x0d, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x0a, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, + 0x73, 0x74, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, + 0x72, 0x52, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x22, 0x20, 0x0a, + 0x0a, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6a, + 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6a, 0x73, 0x6f, 0x6e, 0x22, + 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x62, 0x6f, 0x6c, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x2f, 0x63, 0x68, 0x61, 0x72, 0x6f, 0x6e, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x2f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2f, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cluster/manifestpb/v1/manifest.proto b/cluster/manifestpb/v1/manifest.proto index eac907a68..2ccea05fb 100644 --- a/cluster/manifestpb/v1/manifest.proto +++ b/cluster/manifestpb/v1/manifest.proto @@ -16,6 +16,7 @@ message Cluster { bytes fork_version = 6; // ForkVersion is the fork version (network/chain) of the cluster. It must be 4 bytes. repeated Operator operators = 7; // Operators is the list of operators of the cluster. repeated Validator validators = 8; // Validators is the list of validators of the cluster. + string consensus_protocol = 9; // ConsensusProtocol is the consensus protocol name preferred by the cluster, e.g. "abft". } // Mutation mutates the cluster manifest. diff --git a/cmd/createcluster.go b/cmd/createcluster.go index 7ce564348..14229f206 100644 --- a/cmd/createcluster.go +++ b/cmd/createcluster.go @@ -34,7 +34,7 @@ import ( "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/cluster" "github.com/obolnetwork/charon/core" - "github.com/obolnetwork/charon/core/consensus" + "github.com/obolnetwork/charon/core/consensus/protocols" "github.com/obolnetwork/charon/eth2util" "github.com/obolnetwork/charon/eth2util/deposit" "github.com/obolnetwork/charon/eth2util/enr" @@ -393,7 +393,7 @@ func validateCreateConfig(ctx context.Context, conf clusterConfig) error { return errors.New("number of operators is below minimum", z.Int("operators", conf.NumNodes), z.Int("min", minNodes)) } - if len(conf.ConsensusProtocol) > 0 && !consensus.IsSupportedProtocolName(conf.ConsensusProtocol) { + if len(conf.ConsensusProtocol) > 0 && !protocols.IsSupportedProtocolName(conf.ConsensusProtocol) { return errors.New("unsupported consensus protocol", z.Str("protocol", conf.ConsensusProtocol)) } @@ -960,7 +960,7 @@ func validateDef(ctx context.Context, insecureKeys bool, keymanagerAddrs []strin return errors.New("unsupported network", z.Str("network", network)) } - if len(def.ConsensusProtocol) > 0 && !consensus.IsSupportedProtocolName(def.ConsensusProtocol) { + if len(def.ConsensusProtocol) > 0 && !protocols.IsSupportedProtocolName(def.ConsensusProtocol) { return errors.New("unsupported consensus protocol", z.Str("protocol", def.ConsensusProtocol)) } diff --git a/cmd/createdkg.go b/cmd/createdkg.go index 2c10e2ab6..41923d594 100644 --- a/cmd/createdkg.go +++ b/cmd/createdkg.go @@ -16,7 +16,7 @@ import ( "github.com/obolnetwork/charon/app/version" "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/cluster" - "github.com/obolnetwork/charon/core/consensus" + "github.com/obolnetwork/charon/core/consensus/protocols" "github.com/obolnetwork/charon/eth2util" "github.com/obolnetwork/charon/eth2util/deposit" "github.com/obolnetwork/charon/eth2util/enr" @@ -217,7 +217,7 @@ func validateDKGConfig(numOperators int, network string, depositAmounts []int, c } } - if len(consensusProtocol) > 0 && !consensus.IsSupportedProtocolName(consensusProtocol) { + if len(consensusProtocol) > 0 && !protocols.IsSupportedProtocolName(consensusProtocol) { return errors.New("unsupported consensus protocol", z.Str("protocol", consensusProtocol)) } diff --git a/cmd/run.go b/cmd/run.go index 1bc174f05..99c60b991 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -92,6 +92,7 @@ func bindRunFlags(cmd *cobra.Command, config *app.Config) { cmd.Flags().Int64Var(&config.TestnetConfig.GenesisTimestamp, "testnet-genesis-timestamp", 0, "Genesis timestamp of the custom test network.") cmd.Flags().StringVar(&config.TestnetConfig.CapellaHardFork, "testnet-capella-hard-fork", "", "Capella hard fork version of the custom test network.") cmd.Flags().StringVar(&config.ProcDirectory, "proc-directory", "", "Directory to look into in order to detect other stack components running on the host.") + cmd.Flags().StringVar(&config.ConsensusProtocol, "consensus-protocol", "", "Preferred consensus protocol name for the node. Selected automatically when not specified.") wrapPreRunE(cmd, func(*cobra.Command, []string) error { if len(config.BeaconNodeAddrs) == 0 && !config.SimnetBMock { diff --git a/cmd/version.go b/cmd/version.go index 11f0c9768..c96fb38e6 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/pflag" "github.com/obolnetwork/charon/app/version" - "github.com/obolnetwork/charon/core/consensus" + "github.com/obolnetwork/charon/core/consensus/protocols" ) type versionConfig struct { @@ -66,7 +66,7 @@ func runVersionCmd(out io.Writer, config versionConfig) { _, _ = fmt.Fprint(out, "Consensus protocols:\n") - for _, protocol := range consensus.Protocols() { + for _, protocol := range protocols.Protocols() { _, _ = fmt.Fprintf(out, "\t%v\n", protocol) } } diff --git a/cmd/version_internal_test.go b/cmd/version_internal_test.go index f0139e537..66a038cc4 100644 --- a/cmd/version_internal_test.go +++ b/cmd/version_internal_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/obolnetwork/charon/app/version" - "github.com/obolnetwork/charon/core/consensus" + "github.com/obolnetwork/charon/core/consensus/protocols" ) func TestRunVersionCmd(t *testing.T) { @@ -44,6 +44,6 @@ func TestRunVersionCmd(t *testing.T) { require.Contains(t, str, "Package:") require.Contains(t, str, "Dependencies:") require.Contains(t, str, "Consensus protocols:") - require.Contains(t, str, consensus.Protocols()[0]) + require.Contains(t, str, protocols.Protocols()[0]) }) } diff --git a/core/consensus/controller.go b/core/consensus/controller.go new file mode 100644 index 000000000..af4260700 --- /dev/null +++ b/core/consensus/controller.go @@ -0,0 +1,121 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package consensus + +import ( + "context" + "sync" + + k1 "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/protocol" + + "github.com/obolnetwork/charon/app/errors" + "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/qbft" + "github.com/obolnetwork/charon/p2p" +) + +type DeadlinerFactory func(name string) core.Deadliner + +type consensusController struct { + tcpNode host.Host + sender *p2p.Sender + peers []p2p.Peer + p2pKey *k1.PrivateKey + gaterFunc core.DutyGaterFunc + deadlineFunc core.DeadlineFunc + debugger Debugger + defaultConsensus core.Consensus + wrappedConsensus *consensusWrapper + + mutable struct { + sync.Mutex + cancelWrappedCtx context.CancelFunc + } +} + +// NewConsensusController creates a new consensus controller with the default consensus protocol. +func NewConsensusController(ctx context.Context, tcpNode host.Host, sender *p2p.Sender, + peers []p2p.Peer, p2pKey *k1.PrivateKey, deadlineFunc core.DeadlineFunc, + gaterFunc core.DutyGaterFunc, debugger Debugger, +) (core.ConsensusController, error) { + qbftDeadliner := core.NewDeadliner(ctx, "consensus.qbft", deadlineFunc) + defaultConsensus, err := qbft.NewConsensus(tcpNode, sender, peers, p2pKey, qbftDeadliner, gaterFunc, debugger.AddInstance) + if err != nil { + return nil, err + } + + return &consensusController{ + tcpNode: tcpNode, + sender: sender, + peers: peers, + p2pKey: p2pKey, + gaterFunc: gaterFunc, + deadlineFunc: deadlineFunc, + debugger: debugger, + defaultConsensus: defaultConsensus, + wrappedConsensus: newConsensusWrapper(defaultConsensus), + }, nil +} + +func (f *consensusController) Start(ctx context.Context) { + f.defaultConsensus.Start(ctx) + + go func() { + <-ctx.Done() + + f.mutable.Lock() + defer f.mutable.Unlock() + + if f.mutable.cancelWrappedCtx != nil { + f.mutable.cancelWrappedCtx() + } + }() +} + +// DefaultConsensus returns the default consensus instance. +func (f *consensusController) DefaultConsensus() core.Consensus { + return f.defaultConsensus +} + +// CurrentConsensus returns the current consensus instance. +func (f *consensusController) CurrentConsensus() core.Consensus { + return f.wrappedConsensus +} + +// SetCurrentConsensusForProtocol sets the current consensus instance for the given protocol id. +func (f *consensusController) SetCurrentConsensusForProtocol(_ context.Context, protocol protocol.ID) error { + if f.wrappedConsensus.ProtocolID() == protocol { + return nil + } + + if protocol == f.defaultConsensus.ProtocolID() { + f.wrappedConsensus.SetImpl(f.defaultConsensus) + + return nil + } + + // TODO: When introducing new consensus protocols, add them here as follow: + /* + cctx, cancel := context.WithCancel(ctx) + + f.mutable.Lock() + defer f.mutable.Unlock() + + if f.mutable.cancelWrappedCtx != nil { + // Stopping the previous consensus instance if not the default one. + f.mutable.cancelWrappedCtx() + } + + xyzDeadliner := core.NewDeadliner(cctx, "consensus.xyz", f.deadlineFunc) + xyzConsensus := xyz.NewConsensus(...) + + f.mutable.cancelWrappedCtx = cancel + f.wrappedConsensus.SetImpl(xyzConsensus) + + xyzConsensus.Start(cctx) + */ + + return errors.New("unsupported protocol id") +} diff --git a/core/consensus/controller_test.go b/core/consensus/controller_test.go new file mode 100644 index 000000000..a86ea5dc1 --- /dev/null +++ b/core/consensus/controller_test.go @@ -0,0 +1,81 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package consensus_test + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/libp2p/go-libp2p" + libp2pcrypto "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/cluster" + "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus" + csmocks "github.com/obolnetwork/charon/core/consensus/mocks" + "github.com/obolnetwork/charon/core/consensus/protocols" + "github.com/obolnetwork/charon/eth2util/enr" + "github.com/obolnetwork/charon/p2p" + "github.com/obolnetwork/charon/testutil" +) + +func TestConsensusController(t *testing.T) { + var hosts []host.Host + var peers []p2p.Peer + + seed := 0 + random := rand.New(rand.NewSource(int64(seed))) + lock, p2pkeys, _ := cluster.NewForT(t, 1, 3, 3, seed, random) + + gaterFunc := func(core.Duty) bool { return true } + + for i := range 3 { + addr := testutil.AvailableAddr(t) + mAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", addr.IP, addr.Port)) + require.NoError(t, err) + + priv := (*libp2pcrypto.Secp256k1PrivateKey)(p2pkeys[i]) + h, err := libp2p.New(libp2p.Identity(priv), libp2p.ListenAddrs(mAddr)) + testutil.SkipIfBindErr(t, err) + require.NoError(t, err) + + record, err := enr.Parse(lock.Operators[i].ENR) + require.NoError(t, err) + + p, err := p2p.NewPeerFromENR(record, i) + require.NoError(t, err) + + peers = append(peers, p) + hosts = append(hosts, h) + } + + deadlineFunc := func(core.Duty) (time.Time, bool) { return time.Time{}, false } + debugger := csmocks.NewDebugger(t) + ctx := context.Background() + + controller, err := consensus.NewConsensusController(ctx, hosts[0], new(p2p.Sender), peers, p2pkeys[0], deadlineFunc, gaterFunc, debugger) + require.NoError(t, err) + require.NotNil(t, controller) + + ctx, cancel := context.WithCancel(ctx) + controller.Start(ctx) + defer cancel() + + t.Run("default and current consensus", func(t *testing.T) { + defaultConsensus := controller.DefaultConsensus() + require.NotNil(t, defaultConsensus) + require.EqualValues(t, protocols.QBFTv2ProtocolID, defaultConsensus.ProtocolID()) + require.NotEqual(t, defaultConsensus, controller.CurrentConsensus()) // because the current is wrapped + }) + + t.Run("unsupported protocol id", func(t *testing.T) { + err := controller.SetCurrentConsensusForProtocol(context.TODO(), "boo") + require.ErrorContains(t, err, "unsupported protocol id") + }) +} diff --git a/core/consensus/debugger.go b/core/consensus/debugger.go index 83678a547..09bb477da 100644 --- a/core/consensus/debugger.go +++ b/core/consensus/debugger.go @@ -16,6 +16,8 @@ import ( pbv1 "github.com/obolnetwork/charon/core/corepb/v1" ) +//go:generate mockery --name=Debugger --output=mocks --outpkg=mocks --case=underscore + const maxDebuggerBuffer = 50 * (1 << 20) // 50 MB. // Debugger is an interface for debugging consensus messages. diff --git a/core/consensus/debugger_internal_test.go b/core/consensus/debugger_internal_test.go index 3baa6f711..d51147167 100644 --- a/core/consensus/debugger_internal_test.go +++ b/core/consensus/debugger_internal_test.go @@ -29,7 +29,7 @@ func TestDebugger(t *testing.T) { { Timestamp: timestamppb.Now(), // Eventually the ConsensusMsg will be replaced by a more generic message type. - Msg: &pbv1.ConsensusMsg{ + Msg: &pbv1.QBFTConsensusMsg{ Msg: randomQBFTMsg(), Justification: []*pbv1.QBFTMsg{randomQBFTMsg(), randomQBFTMsg()}, }, diff --git a/core/consensus/metrics.go b/core/consensus/metrics.go deleted file mode 100644 index 8a6eee996..000000000 --- a/core/consensus/metrics.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 - -package consensus - -import ( - "github.com/prometheus/client_golang/prometheus" - - "github.com/obolnetwork/charon/app/promauto" -) - -var ( - // Using gauge since the value changes slowly, once per slot. - decidedRoundsGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "core", - Subsystem: "consensus", - Name: "decided_rounds", - Help: "Number of rounds it took to decide consensus instances by duty and timer type.", - }, []string{"duty", "timer"}) - - // Using gauge since the value changes slowly, once per slot. - decidedLeaderGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "core", - Subsystem: "consensus", - Name: "decided_leader_index", - Help: "Leader node index of the decision round by duty.", - }, []string{"duty"}) - - consensusDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "core", - Subsystem: "consensus", - Name: "duration_seconds", - Help: "Duration of a consensus instance in seconds by duty and timer type.", - Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 20, 30, 60}, - }, []string{"duty", "timer"}) - - consensusTimeout = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "core", - Subsystem: "consensus", - Name: "timeout_total", - Help: "Total count of consensus timeouts by duty and timer type.", - }, []string{"duty", "timer"}) - - consensusError = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "core", - Subsystem: "consensus", - Name: "error_total", - Help: "Total count of consensus errors", - }) -) diff --git a/core/consensus/metrics/metrics.go b/core/consensus/metrics/metrics.go new file mode 100644 index 000000000..31f864d41 --- /dev/null +++ b/core/consensus/metrics/metrics.go @@ -0,0 +1,100 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/obolnetwork/charon/app/promauto" +) + +var ( + decidedRoundsGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "core", + Subsystem: "consensus", + Name: "decided_rounds", + Help: "Number of decided rounds by protocol, duty, and timer", + }, []string{"protocol", "duty", "timer"}) + + decidedLeaderGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "core", + Subsystem: "consensus", + Name: "decided_leader_index", + Help: "Index of the decided leader by protocol and duty", + }, []string{"protocol", "duty"}) + + consensusDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "core", + Subsystem: "consensus", + Name: "duration_seconds", + Help: "Duration of the consensus process by protocol, duty, and timer", + }, []string{"protocol", "duty", "timer"}) + + consensusTimeout = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "core", + Subsystem: "consensus", + Name: "timeout_total", + Help: "Total count of consensus timeouts by protocol, duty, and timer", + }, []string{"protocol", "duty", "timer"}) + + consensusError = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "core", + Subsystem: "consensus", + Name: "error_total", + Help: "Total count of consensus errors by protocol", + }, []string{"protocol"}) +) + +// ConsensusMetrics defines the interface for consensus metrics. +type ConsensusMetrics interface { + // SetDecidedRounds sets the number of decided rounds for a given duty and timer. + SetDecidedRounds(duty, timer string, rounds int64) + + // SetDecidedLeaderIndex sets the decided leader index for a given duty. + SetDecidedLeaderIndex(duty string, leaderIndex int64) + + // ObserveConsensusDuration observes the duration of the consensus process for a given duty and timer. + ObserveConsensusDuration(duty, timer string, duration float64) + + // IncConsensusTimeout increments the consensus timeout counter for a given duty and timer. + IncConsensusTimeout(duty, timer string) + + // IncConsensusError increments the consensus error counter. + IncConsensusError() +} + +type consensusMetrics struct { + protocolID string +} + +// NewConsensusMetrics creates a new instance of ConsensusMetrics with the given protocol ID. +func NewConsensusMetrics(protocolID string) ConsensusMetrics { + return &consensusMetrics{ + protocolID: protocolID, + } +} + +// SetDecidedRounds sets the number of decided rounds for a given duty and timer. +func (m *consensusMetrics) SetDecidedRounds(duty, timer string, rounds int64) { + decidedRoundsGauge.WithLabelValues(m.protocolID, duty, timer).Set(float64(rounds)) +} + +// SetDecidedLeaderIndex sets the decided leader index for a given duty. +func (m *consensusMetrics) SetDecidedLeaderIndex(duty string, leaderIndex int64) { + decidedLeaderGauge.WithLabelValues(m.protocolID, duty).Set(float64(leaderIndex)) +} + +// ObserveConsensusDuration observes the duration of the consensus process for a given duty and timer. +func (m *consensusMetrics) ObserveConsensusDuration(duty, timer string, duration float64) { + consensusDuration.WithLabelValues(m.protocolID, duty, timer).Observe(duration) +} + +// IncConsensusTimeout increments the consensus timeout counter for a given duty and timer. +func (m *consensusMetrics) IncConsensusTimeout(duty, timer string) { + consensusTimeout.WithLabelValues(m.protocolID, duty, timer).Inc() +} + +// IncConsensusError increments the consensus error counter. +func (m *consensusMetrics) IncConsensusError() { + consensusError.WithLabelValues(m.protocolID).Inc() +} diff --git a/core/consensus/metrics/metrics_test.go b/core/consensus/metrics/metrics_test.go new file mode 100644 index 000000000..3db82427d --- /dev/null +++ b/core/consensus/metrics/metrics_test.go @@ -0,0 +1,103 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package metrics_test + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + pb "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/app/promauto" + "github.com/obolnetwork/charon/core/consensus/metrics" +) + +func TestConsensusMetrics_SetDecidedRounds(t *testing.T) { + cm := metrics.NewConsensusMetrics("test") + + cm.SetDecidedRounds("duty", "timer", 1) + + m := gatherMetric(t, "core_consensus_decided_rounds") + require.InEpsilon(t, 1, m.GetMetric()[0].GetGauge().GetValue(), 0.0001) + verifyLabel(t, m.GetMetric()[0].GetLabel(), "protocol", "test") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "duty", "duty") +} + +func TestConsensusMetrics_SetDecidedLeaderIndex(t *testing.T) { + cm := metrics.NewConsensusMetrics("test") + + cm.SetDecidedLeaderIndex("duty", 123) + + m := gatherMetric(t, "core_consensus_decided_leader_index") + require.InEpsilon(t, 123, m.GetMetric()[0].GetGauge().GetValue(), 0.0001) + verifyLabel(t, m.GetMetric()[0].GetLabel(), "protocol", "test") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "duty", "duty") +} + +func TestConsensusMetrics_ObserveConsensusDuration(t *testing.T) { + cm := metrics.NewConsensusMetrics("test") + + cm.ObserveConsensusDuration("duty", "timer", 1) + + m := gatherMetric(t, "core_consensus_duration_seconds") + require.EqualValues(t, 1, m.GetMetric()[0].GetHistogram().GetSampleCount()) + verifyLabel(t, m.GetMetric()[0].GetLabel(), "protocol", "test") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "duty", "duty") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "timer", "timer") +} + +func TestConsensusMetrics_IncConsensusTimeout(t *testing.T) { + cm := metrics.NewConsensusMetrics("test") + + cm.IncConsensusTimeout("duty", "timer") + + m := gatherMetric(t, "core_consensus_timeout_total") + require.InEpsilon(t, 1, m.GetMetric()[0].GetCounter().GetValue(), 0.0001) + verifyLabel(t, m.GetMetric()[0].GetLabel(), "protocol", "test") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "duty", "duty") + verifyLabel(t, m.GetMetric()[0].GetLabel(), "timer", "timer") +} + +func TestConsensusMetrics_IncConsensusError(t *testing.T) { + cm := metrics.NewConsensusMetrics("test") + + cm.IncConsensusError() + + m := gatherMetric(t, "core_consensus_error_total") + require.InEpsilon(t, 1, m.GetMetric()[0].GetCounter().GetValue(), 0.0001) + verifyLabel(t, m.GetMetric()[0].GetLabel(), "protocol", "test") +} + +func gatherMetric(t *testing.T, name string) *pb.MetricFamily { + t.Helper() + + registry, err := promauto.NewRegistry(prometheus.Labels{}) + require.NoError(t, err) + + mfa, err := registry.Gather() + require.NoError(t, err) + + for _, mf := range mfa { + if mf.GetName() == name { + return mf + } + } + + require.Fail(t, "metric not found") + + return nil +} + +func verifyLabel(t *testing.T, labels []*pb.LabelPair, name, value string) { + t.Helper() + + for _, label := range labels { + if label.GetName() == name { + require.Equal(t, value, label.GetValue()) + return + } + } + + require.Fail(t, "label not found") +} diff --git a/core/consensus/mocks/debugger.go b/core/consensus/mocks/debugger.go new file mode 100644 index 000000000..f3cca1213 --- /dev/null +++ b/core/consensus/mocks/debugger.go @@ -0,0 +1,41 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +// Code generated by mockery v2.42.1. DO NOT EDIT. + +package mocks + +import ( + http "net/http" + + v1 "github.com/obolnetwork/charon/core/corepb/v1" + mock "github.com/stretchr/testify/mock" +) + +// Debugger is an autogenerated mock type for the Debugger type +type Debugger struct { + mock.Mock +} + +// AddInstance provides a mock function with given fields: instance +func (_m *Debugger) AddInstance(instance *v1.SniffedConsensusInstance) { + _m.Called(instance) +} + +// ServeHTTP provides a mock function with given fields: _a0, _a1 +func (_m *Debugger) ServeHTTP(_a0 http.ResponseWriter, _a1 *http.Request) { + _m.Called(_a0, _a1) +} + +// NewDebugger creates a new instance of Debugger. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewDebugger(t interface { + mock.TestingT + Cleanup(func()) +}) *Debugger { + mock := &Debugger{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/consensus/protocols/protocols.go b/core/consensus/protocols/protocols.go new file mode 100644 index 000000000..961ae92ef --- /dev/null +++ b/core/consensus/protocols/protocols.go @@ -0,0 +1,61 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package protocols + +import ( + "strings" + + "github.com/libp2p/go-libp2p/core/protocol" +) + +const ( + protocolIDPrefix = "/charon/consensus/" + + QBFTv2ProtocolID = "/charon/consensus/qbft/2.0.0" +) + +// Protocols returns the supported protocols of this package in order of precedence. +func Protocols() []protocol.ID { + return []protocol.ID{QBFTv2ProtocolID} +} + +// MostPreferredConsensusProtocol returns the most preferred consensus protocol from the given list. +func MostPreferredConsensusProtocol(protocols []string) string { + for _, p := range protocols { + if strings.HasPrefix(p, protocolIDPrefix) { + return p + } + } + + return QBFTv2ProtocolID +} + +// IsSupportedProtocolName returns true if the protocol name is supported. +func IsSupportedProtocolName(name string) bool { + for _, p := range Protocols() { + nameAndVersion := strings.TrimPrefix(string(p), protocolIDPrefix) + parts := strings.Split(nameAndVersion, "/") + if len(parts) > 0 && parts[0] == strings.ToLower(name) { + return true + } + } + + return false +} + +// PrioritizeProtocolsByName bumps given protocols priority by protocol name. +// The initial order of the protocols and versions is preserved. +func PrioritizeProtocolsByName(protocolName string, allProtocols []protocol.ID) []protocol.ID { + targetPrefix := protocolIDPrefix + protocolName + "/" + + var bumped, others []protocol.ID + for _, p := range allProtocols { + if strings.HasPrefix(string(p), targetPrefix) { + bumped = append(bumped, p) + } else { + others = append(others, p) + } + } + + return append(bumped, others...) +} diff --git a/core/consensus/protocols/protocols_test.go b/core/consensus/protocols/protocols_test.go new file mode 100644 index 000000000..31321ddd6 --- /dev/null +++ b/core/consensus/protocols/protocols_test.go @@ -0,0 +1,56 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package protocols_test + +import ( + "testing" + + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/core/consensus/protocols" +) + +func TestIsSupportedProtocolName(t *testing.T) { + require.True(t, protocols.IsSupportedProtocolName("qbft")) + require.False(t, protocols.IsSupportedProtocolName("unreal")) +} + +func TestProtocols(t *testing.T) { + require.Equal(t, []protocol.ID{ + protocols.QBFTv2ProtocolID, + }, protocols.Protocols()) +} + +func TestMostPreferredConsensusProtocol(t *testing.T) { + t.Run("default is qbft", func(t *testing.T) { + require.Equal(t, protocols.QBFTv2ProtocolID, protocols.MostPreferredConsensusProtocol([]string{"unreal"})) + require.Equal(t, protocols.QBFTv2ProtocolID, protocols.MostPreferredConsensusProtocol([]string{})) + }) + + t.Run("latest abft is preferred", func(t *testing.T) { + pp := []string{ + "/charon/consensus/abft/3.0.0", + "/charon/consensus/abft/1.0.0", + "/charon/consensus/qbft/1.0.0", + } + require.Equal(t, "/charon/consensus/abft/3.0.0", protocols.MostPreferredConsensusProtocol(pp)) + }) +} + +func TestPrioritizeProtocolsByName(t *testing.T) { + intitial := []protocol.ID{ + "/charon/consensus/hotstuff/1.0.0", + "/charon/consensus/abft/3.0.0", + "/charon/consensus/abft/1.0.0", + "/charon/consensus/qbft/1.0.0", + } + + bumped := protocols.PrioritizeProtocolsByName("abft", intitial) + require.Equal(t, []protocol.ID{ + "/charon/consensus/abft/3.0.0", + "/charon/consensus/abft/1.0.0", + "/charon/consensus/hotstuff/1.0.0", + "/charon/consensus/qbft/1.0.0", + }, bumped) +} diff --git a/core/consensus/msg.go b/core/consensus/qbft/msg.go similarity index 81% rename from core/consensus/msg.go rename to core/consensus/qbft/msg.go index a4d1352ed..1822cabfe 100644 --- a/core/consensus/msg.go +++ b/core/consensus/qbft/msg.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( k1 "github.com/decred/dcrd/dcrec/secp256k1/v4" @@ -15,10 +15,10 @@ import ( "github.com/obolnetwork/charon/core/qbft" ) -// newMsg returns a new msg. -func newMsg(pbMsg *pbv1.QBFTMsg, justification []*pbv1.QBFTMsg, values map[[32]byte]*anypb.Any) (msg, error) { +// newMsg returns a new QBFT Msg. +func newMsg(pbMsg *pbv1.QBFTMsg, justification []*pbv1.QBFTMsg, values map[[32]byte]*anypb.Any) (Msg, error) { if pbMsg == nil { - return msg{}, errors.New("nil qbft message") + return Msg{}, errors.New("nil qbft message") } // Do all possible error conversions first. @@ -30,14 +30,14 @@ func newMsg(pbMsg *pbv1.QBFTMsg, justification []*pbv1.QBFTMsg, values map[[32]b if hash, ok := toHash32(pbMsg.GetValueHash()); ok { valueHash = hash if _, ok := values[valueHash]; !ok { - return msg{}, errors.New("value hash not found in values") + return Msg{}, errors.New("value hash not found in values") } } if hash, ok := toHash32(pbMsg.GetPreparedValueHash()); ok { preparedValueHash = hash if _, ok := values[preparedValueHash]; !ok { - return msg{}, errors.New("prepared value hash not found in values") + return Msg{}, errors.New("prepared value hash not found in values") } } @@ -45,13 +45,13 @@ func newMsg(pbMsg *pbv1.QBFTMsg, justification []*pbv1.QBFTMsg, values map[[32]b for _, j := range justification { impl, err := newMsg(j, nil, values) if err != nil { - return msg{}, err + return Msg{}, err } justImpls = append(justImpls, impl) } - return msg{ + return Msg{ msg: pbMsg, valueHash: valueHash, values: values, @@ -61,8 +61,8 @@ func newMsg(pbMsg *pbv1.QBFTMsg, justification []*pbv1.QBFTMsg, values map[[32]b }, nil } -// msg wraps *pbv1.QBFTMsg and justifications and implements qbft.Msg[core.Duty, [32]byte]. -type msg struct { +// Msg wraps *pbv1.QBFTMsg and justifications and implements qbft.Msg[core.Duty, [32]byte]. +type Msg struct { msg *pbv1.QBFTMsg valueHash [32]byte preparedValueHash [32]byte @@ -72,45 +72,53 @@ type msg struct { justification []qbft.Msg[core.Duty, [32]byte] } -func (m msg) Type() qbft.MsgType { +func (m Msg) Type() qbft.MsgType { return qbft.MsgType(m.msg.GetType()) } -func (m msg) Instance() core.Duty { +func (m Msg) Instance() core.Duty { return core.DutyFromProto(m.msg.GetDuty()) } -func (m msg) Source() int64 { +func (m Msg) Source() int64 { return m.msg.GetPeerIdx() } -func (m msg) Round() int64 { +func (m Msg) Round() int64 { return m.msg.GetRound() } -func (m msg) Value() [32]byte { +func (m Msg) Value() [32]byte { return m.valueHash } -func (m msg) PreparedRound() int64 { +func (m Msg) Values() map[[32]byte]*anypb.Any { + return m.values +} + +func (m Msg) Msg() *pbv1.QBFTMsg { + return m.msg +} + +func (m Msg) PreparedRound() int64 { return m.msg.GetPreparedRound() } -func (m msg) PreparedValue() [32]byte { +func (m Msg) PreparedValue() [32]byte { return m.preparedValueHash } -func (m msg) Justification() []qbft.Msg[core.Duty, [32]byte] { +func (m Msg) Justification() []qbft.Msg[core.Duty, [32]byte] { return m.justification } -func (m msg) ToConsensusMsg() *pbv1.ConsensusMsg { +func (m Msg) ToConsensusMsg() *pbv1.QBFTConsensusMsg { var values []*anypb.Any for _, v := range m.values { values = append(values, v) } - return &pbv1.ConsensusMsg{ + return &pbv1.QBFTConsensusMsg{ Msg: m.msg, Justification: m.justificationProtos, Values: values, @@ -205,4 +213,4 @@ func toHash32(val []byte) ([32]byte, bool) { return resp, true } -var _ qbft.Msg[core.Duty, [32]byte] = msg{} // Interface assertion +var _ qbft.Msg[core.Duty, [32]byte] = Msg{} // Interface assertion diff --git a/core/consensus/msg_internal_test.go b/core/consensus/qbft/msg_internal_test.go similarity index 86% rename from core/consensus/msg_internal_test.go rename to core/consensus/qbft/msg_internal_test.go index ec5c95777..1750c87c7 100644 --- a/core/consensus/msg_internal_test.go +++ b/core/consensus/qbft/msg_internal_test.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( "encoding/hex" @@ -15,7 +15,7 @@ import ( "github.com/obolnetwork/charon/core" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" - "github.com/obolnetwork/charon/core/qbft" + coreqbft "github.com/obolnetwork/charon/core/qbft" "github.com/obolnetwork/charon/testutil" ) @@ -43,7 +43,7 @@ func TestSigning(t *testing.T) { privkey, err := k1.GeneratePrivateKey() require.NoError(t, err) - msg := randomMsg(t) + msg := newRandomQBFTMsg(t) signed, err := signMsg(msg, privkey) require.NoError(t, err) @@ -78,7 +78,7 @@ func TestNewMsg(t *testing.T) { } msg, err := newMsg(&pbv1.QBFTMsg{ - Type: int64(qbft.MsgPrePrepare), + Type: int64(coreqbft.MsgPrePrepare), ValueHash: hash1[:], PreparedValueHash: hash2[:], }, nil, values) @@ -86,7 +86,7 @@ func TestNewMsg(t *testing.T) { require.Equal(t, msg.Value(), hash1) require.Equal(t, msg.PreparedValue(), hash2) - require.EqualValues(t, msg.values, values) + require.EqualValues(t, msg.Values(), values) } func TestPartialLegacyNewMsg(t *testing.T) { @@ -95,21 +95,21 @@ func TestPartialLegacyNewMsg(t *testing.T) { require.NoError(t, err) _, err = newMsg(&pbv1.QBFTMsg{ - Type: int64(qbft.MsgPrePrepare), + Type: int64(coreqbft.MsgPrePrepare), }, []*pbv1.QBFTMsg{ { - Type: int64(qbft.MsgPrePrepare), + Type: int64(coreqbft.MsgPrePrepare), ValueHash: hash1[:], }, }, make(map[[32]byte]*anypb.Any)) require.ErrorContains(t, err, "value hash not found in values") } -// randomMsg returns a random qbft message. -func randomMsg(t *testing.T) *pbv1.QBFTMsg { +// NewRandomMsgForT returns a random qbft message. +func newRandomQBFTMsg(t *testing.T) *pbv1.QBFTMsg { t.Helper() - msgType := 1 + rand.Int63n(int64(qbft.MsgDecided)) + msgType := 1 + rand.Int63n(int64(coreqbft.MsgDecided)) if msgType == 0 { msgType = 1 } diff --git a/core/consensus/component.go b/core/consensus/qbft/qbft.go similarity index 75% rename from core/consensus/component.go rename to core/consensus/qbft/qbft.go index 0fcf35d9f..2096b38e6 100644 --- a/core/consensus/component.go +++ b/core/consensus/qbft/qbft.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( "context" @@ -22,38 +22,18 @@ import ( "github.com/obolnetwork/charon/app/log" "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/metrics" + "github.com/obolnetwork/charon/core/consensus/protocols" + "github.com/obolnetwork/charon/core/consensus/utils" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" "github.com/obolnetwork/charon/core/qbft" "github.com/obolnetwork/charon/p2p" ) -const ( - recvBuffer = 100 // Allow buffering some initial messages when this node is late to start an instance. - protocolID2 = "/charon/consensus/qbft/2.0.0" -) - -// Protocols returns the supported protocols of this package in order of precedence. -func Protocols() []protocol.ID { - return []protocol.ID{protocolID2} -} - -// IsSupportedProtocolName returns true if the protocol name is supported. -func IsSupportedProtocolName(name string) bool { - for _, p := range Protocols() { - nameAndVersion := strings.TrimPrefix(string(p), "/charon/consensus/") - parts := strings.Split(nameAndVersion, "/") - if len(parts) > 0 && parts[0] == strings.ToLower(name) { - return true - } - } - - return false -} - type subscriber func(ctx context.Context, duty core.Duty, value proto.Message) error // newDefinition returns a qbft definition (this is constant across all consensus instances). -func newDefinition(nodes int, subs func() []subscriber, roundTimer roundTimer, +func newDefinition(nodes int, subs func() []subscriber, roundTimer utils.RoundTimer, decideCallback func(qcommit []qbft.Msg[core.Duty, [32]byte]), ) qbft.Definition[core.Duty, [32]byte] { quorum := qbft.Definition[int, int]{Nodes: nodes}.Quorum() @@ -67,13 +47,13 @@ func newDefinition(nodes int, subs func() []subscriber, roundTimer roundTimer, // Decide sends consensus output to subscribers. Decide: func(ctx context.Context, duty core.Duty, _ [32]byte, qcommit []qbft.Msg[core.Duty, [32]byte]) { defer endCtxSpan(ctx) // End the parent tracing span when decided - msg, ok := qcommit[0].(msg) + msg, ok := qcommit[0].(Msg) if !ok { log.Error(ctx, "Invalid message type", nil) return } - anyValue, ok := msg.values[msg.valueHash] + anyValue, ok := msg.Values()[msg.Value()] if !ok { log.Error(ctx, "Invalid value hash", nil) return @@ -135,80 +115,14 @@ func newDefinition(nodes int, subs func() []subscriber, roundTimer roundTimer, Nodes: nodes, // FIFOLimit caps the max buffered messages per peer. - FIFOLimit: recvBuffer, - } -} - -// newInstanceIO returns a new instanceIO. -func newInstanceIO() instanceIO { - return instanceIO{ - participated: make(chan struct{}), - proposed: make(chan struct{}), - running: make(chan struct{}), - recvBuffer: make(chan msg, recvBuffer), - hashCh: make(chan [32]byte, 1), - valueCh: make(chan proto.Message, 1), - errCh: make(chan error, 1), - decidedAtCh: make(chan time.Time, 1), + FIFOLimit: utils.RecvBufferSize, } } -// instanceIO defines the async input and output channels of a -// single consensus instance in the Component. -type instanceIO struct { - participated chan struct{} // Closed when Participate was called for this instance. - proposed chan struct{} // Closed when Propose was called for this instance. - running chan struct{} // Closed when runInstance was already called. - recvBuffer chan msg // Outer receive buffers. - hashCh chan [32]byte // Async input hash channel. - valueCh chan proto.Message // Async input value channel. - errCh chan error // Async output error channel. - decidedAtCh chan time.Time // Async output decided timestamp channel. -} - -// MarkParticipated marks the instance as participated. -// It returns an error if the instance was already marked as participated. -func (io instanceIO) MarkParticipated() error { - select { - case <-io.participated: - return errors.New("already participated") - default: - close(io.participated) - } - - return nil -} - -// MarkProposed marks the instance as proposed. -// It returns an error if the instance was already marked as proposed. -func (io instanceIO) MarkProposed() error { - select { - case <-io.proposed: - return errors.New("already proposed") - default: - close(io.proposed) - } - - return nil -} - -// MaybeStart returns true if the instance wasn't running and has been started by this call, -// otherwise it returns false if the instance was started in the past and is either running now or has completed. -func (io instanceIO) MaybeStart() bool { - select { - case <-io.running: - return false - default: - close(io.running) - } - - return true -} - -// New returns a new consensus QBFT component. -func New(tcpNode host.Host, sender *p2p.Sender, peers []p2p.Peer, p2pKey *k1.PrivateKey, +// NewConsensus returns a new consensus QBFT component. +func NewConsensus(tcpNode host.Host, sender *p2p.Sender, peers []p2p.Peer, p2pKey *k1.PrivateKey, deadliner core.Deadliner, gaterFunc core.DutyGaterFunc, snifferFunc func(*pbv1.SniffedConsensusInstance), -) (*Component, error) { +) (*Consensus, error) { // Extract peer pubkeys. keys := make(map[int64]*k1.PublicKey) var labels []string @@ -223,7 +137,7 @@ func New(tcpNode host.Host, sender *p2p.Sender, peers []p2p.Peer, p2pKey *k1.Pri keys[int64(i)] = pk } - c := &Component{ + c := &Consensus{ tcpNode: tcpNode, sender: sender, peers: peers, @@ -234,15 +148,16 @@ func New(tcpNode host.Host, sender *p2p.Sender, peers []p2p.Peer, p2pKey *k1.Pri snifferFunc: snifferFunc, gaterFunc: gaterFunc, dropFilter: log.Filter(), - timerFunc: getTimerFunc(), + timerFunc: utils.GetTimerFunc(), + metrics: metrics.NewConsensusMetrics(protocols.QBFTv2ProtocolID), } - c.mutable.instances = make(map[core.Duty]instanceIO) + c.mutable.instances = make(map[core.Duty]*utils.InstanceIO[Msg]) return c, nil } -// Component implements core.Consensus. -type Component struct { +// Consensus implements core.Consensus & priority.coreConsensus. +type Consensus struct { // Immutable state tcpNode host.Host sender *p2p.Sender @@ -255,18 +170,24 @@ type Component struct { snifferFunc func(*pbv1.SniffedConsensusInstance) gaterFunc core.DutyGaterFunc dropFilter z.Field // Filter buffer overflow errors (possible DDoS) - timerFunc timerFunc + timerFunc utils.TimerFunc + metrics metrics.ConsensusMetrics // Mutable state mutable struct { sync.Mutex - instances map[core.Duty]instanceIO + instances map[core.Duty]*utils.InstanceIO[Msg] } } +// ProtocolID returns the protocol ID. +func (*Consensus) ProtocolID() protocol.ID { + return protocols.QBFTv2ProtocolID +} + // Subscribe registers a callback for unsigned duty data proposals from leaders. // Note this function is not thread safe, it should be called *before* Start and Propose. -func (c *Component) Subscribe(fn func(ctx context.Context, duty core.Duty, set core.UnsignedDataSet) error) { +func (c *Consensus) Subscribe(fn func(ctx context.Context, duty core.Duty, set core.UnsignedDataSet) error) { c.subs = append(c.subs, func(ctx context.Context, duty core.Duty, value proto.Message) error { unsignedPB, ok := value.(*pbv1.UnsignedDataSet) if !ok { @@ -283,13 +204,13 @@ func (c *Component) Subscribe(fn func(ctx context.Context, duty core.Duty, set c } // subscribers returns the subscribers. -func (c *Component) subscribers() []subscriber { +func (c *Consensus) subscribers() []subscriber { return c.subs } // SubscribePriority registers a callback for priority protocol message proposals from leaders. // Note this function is not thread safe, it should be called *before* Start and Propose. -func (c *Component) SubscribePriority(fn func(ctx context.Context, duty core.Duty, msg *pbv1.PriorityResult) error) { +func (c *Consensus) SubscribePriority(fn func(ctx context.Context, duty core.Duty, msg *pbv1.PriorityResult) error) { c.subs = append(c.subs, func(ctx context.Context, duty core.Duty, value proto.Message) error { msg, ok := value.(*pbv1.PriorityResult) if !ok { @@ -300,16 +221,17 @@ func (c *Component) SubscribePriority(fn func(ctx context.Context, duty core.Dut }) } -// Start registers the libp2p receive handler and starts a goroutine that cleans state. This should only be called once. -func (c *Component) Start(ctx context.Context) { - p2p.RegisterHandler("qbft", c.tcpNode, protocolID2, - func() proto.Message { return new(pbv1.ConsensusMsg) }, +// Start registers libp2p handler and runs internal routines until the context is cancelled. +func (c *Consensus) Start(ctx context.Context) { + p2p.RegisterHandler("qbft", c.tcpNode, protocols.QBFTv2ProtocolID, + func() proto.Message { return new(pbv1.QBFTConsensusMsg) }, c.handle) go func() { for { select { case <-ctx.Done(): + // No need to unregister QBFT handler. return case duty := <-c.deadliner.C(): c.deleteInstanceIO(duty) @@ -322,7 +244,7 @@ func (c *Component) Start(ctx context.Context) { // It either runs the consensus instance if it is not already running or // waits until it completes, in both cases it returns the resulting error. // Note this errors if called multiple times for the same duty. -func (c *Component) Propose(ctx context.Context, duty core.Duty, data core.UnsignedDataSet) error { +func (c *Consensus) Propose(ctx context.Context, duty core.Duty, data core.UnsignedDataSet) error { // Hash the proposed data, since qbft only supports simple comparable values. value, err := core.UnsignedDataSetToProto(data) if err != nil { @@ -336,7 +258,7 @@ func (c *Component) Propose(ctx context.Context, duty core.Duty, data core.Unsig // It either runs the consensus instance if it is not already running or // waits until it completes, in both cases it returns the resulting error. // Note this errors if called multiple times for the same duty. -func (c *Component) ProposePriority(ctx context.Context, duty core.Duty, msg *pbv1.PriorityResult) error { +func (c *Consensus) ProposePriority(ctx context.Context, duty core.Duty, msg *pbv1.PriorityResult) error { return c.propose(ctx, duty, msg) } @@ -344,7 +266,7 @@ func (c *Component) ProposePriority(ctx context.Context, duty core.Duty, msg *pb // It either runs the consensus instance if it is not already running or // waits until it completes, in both cases it returns the resulting error. // Note this errors if called multiple times for the same duty. -func (c *Component) propose(ctx context.Context, duty core.Duty, value proto.Message) error { +func (c *Consensus) propose(ctx context.Context, duty core.Duty, value proto.Message) error { hash, err := hashProto(value) if err != nil { return err @@ -358,13 +280,13 @@ func (c *Component) propose(ctx context.Context, duty core.Duty, value proto.Mes // Provide proposal inputs to the instance. select { - case inst.valueCh <- value: + case inst.ValueCh <- value: default: return errors.New("input channel full") } select { - case inst.hashCh <- hash: + case inst.HashCh <- hash: default: return errors.New("input channel full") } @@ -373,16 +295,16 @@ func (c *Component) propose(ctx context.Context, duty core.Duty, value proto.Mes proposedAt := time.Now() defer func() { select { - case decidedAt := <-inst.decidedAtCh: + case decidedAt := <-inst.DecidedAtCh: timerType := c.timerFunc(duty).Type() duration := decidedAt.Sub(proposedAt) - consensusDuration.WithLabelValues(duty.Type.String(), string(timerType)).Observe(duration.Seconds()) + c.metrics.ObserveConsensusDuration(duty.Type.String(), string(timerType), duration.Seconds()) default: } }() if !inst.MaybeStart() { // Participate was already called, instance is running. - return <-inst.errCh + return <-inst.ErrCh } return c.runInstance(ctx, duty) @@ -392,7 +314,7 @@ func (c *Component) propose(ctx context.Context, duty core.Duty, value proto.Mes // unsigned data from beacon node and Propose not already called. // Note Propose must still be called for this peer to propose a value when leading a round. // Note this errors if called multiple times for the same duty. -func (c *Component) Participate(ctx context.Context, duty core.Duty) error { +func (c *Consensus) Participate(ctx context.Context, duty core.Duty) error { if duty.Type == core.DutyAggregator || duty.Type == core.DutySyncContribution { return nil // No consensus participate for potential no-op aggregation duties. } @@ -414,10 +336,26 @@ func (c *Component) Participate(ctx context.Context, duty core.Duty) error { return c.runInstance(ctx, duty) } +// Broadcast implements Broadcaster interface. +func (c *Consensus) Broadcast(ctx context.Context, msg *pbv1.QBFTConsensusMsg) error { + for _, peer := range c.peers { + if peer.ID == c.tcpNode.ID() { + // Do not broadcast to self + continue + } + + if err := c.sender.SendAsync(ctx, c.tcpNode, protocols.QBFTv2ProtocolID, peer.ID, msg); err != nil { + return err + } + } + + return nil +} + // runInstance blocks and runs a consensus instance for the given duty. // It returns an error or nil when the context is cancelled. // Note each instance may only be run once. -func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) { +func (c *Consensus) runInstance(ctx context.Context, duty core.Duty) (err error) { roundTimer := c.timerFunc(duty) ctx = log.WithTopic(ctx, "qbft") ctx = log.WithCtx(ctx, z.Any("duty", duty)) @@ -431,7 +369,7 @@ func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) inst := c.getInstanceIO(duty) defer func() { - inst.errCh <- err // Send resulting error to errCh. + inst.ErrCh <- err // Send resulting error to errCh. }() if !c.deadliner.Add(duty) { @@ -453,8 +391,7 @@ func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) decideCallback := func(qcommit []qbft.Msg[core.Duty, [32]byte]) { round := qcommit[0].Round() decided = true - decidedRoundsGauge.WithLabelValues(duty.Type.String(), string(roundTimer.Type())).Set(float64(round)) - inst.decidedAtCh <- time.Now() + inst.DecidedAtCh <- time.Now() leaderIndex := leader(duty, round, nodes) leaderName := c.peers[leaderIndex].Name @@ -465,24 +402,19 @@ func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) z.I64("leader_index", leaderIndex), z.Str("leader_name", leaderName)) - decidedLeaderGauge.WithLabelValues(duty.Type.String()).Set(float64(leaderIndex)) + c.metrics.SetDecidedLeaderIndex(duty.Type.String(), leaderIndex) + c.metrics.SetDecidedRounds(duty.Type.String(), string(roundTimer.Type()), round) } // Create a new qbft definition for this instance. - def := newDefinition(nodes, c.subscribers, roundTimer, decideCallback) + def := newDefinition(len(c.peers), c.subscribers, roundTimer, decideCallback) // Create a new transport that handles sending and receiving for this instance. - t := transport{ - component: c, - values: make(map[[32]byte]*anypb.Any), - valueCh: inst.valueCh, - recvBuffer: make(chan qbft.Msg[core.Duty, [32]byte]), - sniffer: newSniffer(int64(def.Nodes), peerIdx), - } + t := newTransport(c, c.privkey, inst.ValueCh, make(chan qbft.Msg[core.Duty, [32]byte]), newSniffer(int64(def.Nodes), peerIdx)) // Provide sniffed buffer to snifferFunc at the end. defer func() { - c.snifferFunc(t.sniffer.Instance()) + c.snifferFunc(t.SnifferInstance()) }() // Start a receiving goroutine. @@ -491,18 +423,18 @@ func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) // Create a qbft transport from the transport qt := qbft.Transport[core.Duty, [32]byte]{ Broadcast: t.Broadcast, - Receive: t.recvBuffer, + Receive: t.RecvBuffer(), } // Run the algo, blocking until the context is cancelled. - err = qbft.Run(ctx, def, qt, duty, peerIdx, inst.hashCh) + err = qbft.Run(ctx, def, qt, duty, peerIdx, inst.HashCh) if err != nil && !isContextErr(err) { - consensusError.Inc() + c.metrics.IncConsensusError() return err // Only return non-context errors. } if !decided { - consensusTimeout.WithLabelValues(duty.Type.String(), string(roundTimer.Type())).Inc() + c.metrics.IncConsensusTimeout(duty.Type.String(), string(roundTimer.Type())) return errors.New("consensus timeout", z.Str("duty", duty.String())) } @@ -511,10 +443,10 @@ func (c *Component) runInstance(ctx context.Context, duty core.Duty) (err error) } // handle processes an incoming consensus wire message. -func (c *Component) handle(ctx context.Context, _ peer.ID, req proto.Message) (proto.Message, bool, error) { +func (c *Consensus) handle(ctx context.Context, _ peer.ID, req proto.Message) (proto.Message, bool, error) { t0 := time.Now() - pbMsg, ok := req.(*pbv1.ConsensusMsg) + pbMsg, ok := req.(*pbv1.QBFTConsensusMsg) if !ok || pbMsg == nil { return nil, false, errors.New("invalid consensus message") } @@ -576,37 +508,35 @@ func (c *Component) handle(ctx context.Context, _ peer.ID, req proto.Message) (p } // getRecvBuffer returns a receive buffer for the duty. -func (c *Component) getRecvBuffer(duty core.Duty) chan msg { +func (c *Consensus) getRecvBuffer(duty core.Duty) chan Msg { c.mutable.Lock() defer c.mutable.Unlock() inst, ok := c.mutable.instances[duty] if !ok { - inst = newInstanceIO() + inst = utils.NewInstanceIO[Msg]() c.mutable.instances[duty] = inst } - return inst.recvBuffer + return inst.RecvBuffer } -// getInstanceIO returns the duty's instance and true if it were previously created. -func (c *Component) getInstanceIO(duty core.Duty) instanceIO { +// getInstanceIO returns the duty's instance if it were previously created. +func (c *Consensus) getInstanceIO(duty core.Duty) *utils.InstanceIO[Msg] { c.mutable.Lock() defer c.mutable.Unlock() inst, ok := c.mutable.instances[duty] if !ok { // Create new instanceIO. - inst = newInstanceIO() + inst = utils.NewInstanceIO[Msg]() c.mutable.instances[duty] = inst - - return inst } return inst } // deleteInstanceIO deletes the instanceIO for the duty. -func (c *Component) deleteInstanceIO(duty core.Duty) { +func (c *Consensus) deleteInstanceIO(duty core.Duty) { c.mutable.Lock() defer c.mutable.Unlock() @@ -614,7 +544,7 @@ func (c *Component) deleteInstanceIO(duty core.Duty) { } // getPeerIdx returns the local peer index. -func (c *Component) getPeerIdx() (int64, error) { +func (c *Consensus) getPeerIdx() (int64, error) { peerIdx := int64(-1) for i, p := range c.peers { if c.tcpNode.ID() == p.ID { @@ -777,6 +707,7 @@ func leader(duty core.Duty, round int64, nodes int) int64 { return (int64(duty.Slot) + int64(duty.Type) + round) % int64(nodes) } +// valuesByHash returns a map of values by hash. func valuesByHash(values []*anypb.Any) (map[[32]byte]*anypb.Any, error) { resp := make(map[[32]byte]*anypb.Any) for _, v := range values { diff --git a/core/consensus/component_internal_test.go b/core/consensus/qbft/qbft_internal_test.go similarity index 85% rename from core/consensus/component_internal_test.go rename to core/consensus/qbft/qbft_internal_test.go index fc827042d..68524ed36 100644 --- a/core/consensus/component_internal_test.go +++ b/core/consensus/qbft/qbft_internal_test.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( "bytes" @@ -8,12 +8,15 @@ import ( "testing" k1 "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" "github.com/obolnetwork/charon/app/k1util" "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/utils" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" + coremocks "github.com/obolnetwork/charon/core/mocks" "github.com/obolnetwork/charon/core/qbft" "github.com/obolnetwork/charon/testutil" ) @@ -118,15 +121,15 @@ func (t testMsg) Justification() []qbft.Msg[core.Duty, [32]byte] { panic("implement me") } -func TestComponent_handle(t *testing.T) { +func TestQBFTConsensus_handle(t *testing.T) { tests := []struct { name string - mutate func(base *pbv1.ConsensusMsg, c *Component) + mutate func(base *pbv1.QBFTConsensusMsg, c *Consensus) checkErr func(err error) }{ { "qbft message with no pubkey errors", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { // construct a valid basis message signature base.Msg.Duty.Type = 1 base.Msg.Signature = bytes.Repeat([]byte{42}, 65) @@ -142,7 +145,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message with justifications mentioning unknown peerIdx errors", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { p2pKey := testutil.GenerateInsecureK1Key(t, 0) c.pubkeys = make(map[int64]*k1.PublicKey) c.pubkeys[0] = p2pKey.PubKey() @@ -165,7 +168,7 @@ func TestComponent_handle(t *testing.T) { // construct a justification base.Justification = []*pbv1.QBFTMsg{ - randomMsg(t), + newRandomQBFTMsg(t), } base.Justification[0].PeerIdx = 42 @@ -189,7 +192,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message with nil justification present in slice", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { p2pKey := testutil.GenerateInsecureK1Key(t, 0) c.pubkeys = make(map[int64]*k1.PublicKey) c.pubkeys[0] = p2pKey.PubKey() @@ -222,7 +225,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message values present but nil", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { p2pKey := testutil.GenerateInsecureK1Key(t, 0) c.pubkeys = make(map[int64]*k1.PublicKey) c.pubkeys[0] = p2pKey.PubKey() @@ -255,7 +258,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message with invalid duty fails", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { // construct a valid basis message signature base.Msg.Duty.Type = 1 base.Msg.Signature = bytes.Repeat([]byte{42}, 65) @@ -271,7 +274,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message with valid duty fails because justification has different duty type", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { p2pKey := testutil.GenerateInsecureK1Key(t, 0) c.pubkeys = make(map[int64]*k1.PublicKey) c.pubkeys[0] = p2pKey.PubKey() @@ -294,7 +297,7 @@ func TestComponent_handle(t *testing.T) { // construct a justification base.Justification = []*pbv1.QBFTMsg{ - randomMsg(t), + newRandomQBFTMsg(t), } base.Justification[0].PeerIdx = 0 @@ -318,7 +321,7 @@ func TestComponent_handle(t *testing.T) { }, { "qbft message with valid duty and justification with same duty does not fail", - func(base *pbv1.ConsensusMsg, c *Component) { + func(base *pbv1.QBFTConsensusMsg, c *Consensus) { p2pKey := testutil.GenerateInsecureK1Key(t, 0) c.pubkeys = make(map[int64]*k1.PublicKey) c.pubkeys[0] = p2pKey.PubKey() @@ -341,7 +344,7 @@ func TestComponent_handle(t *testing.T) { // construct a justification base.Justification = []*pbv1.QBFTMsg{ - randomMsg(t), + newRandomQBFTMsg(t), } base.Justification[0].PeerIdx = 0 @@ -369,13 +372,15 @@ func TestComponent_handle(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - var tc Component - tc.deadliner = testDeadliner{} - tc.mutable.instances = make(map[core.Duty]instanceIO) + var tc Consensus + deadliner := coremocks.NewDeadliner(t) + deadliner.On("Add", mock.Anything).Maybe().Return(true) + tc.deadliner = deadliner + tc.mutable.instances = make(map[core.Duty]*utils.InstanceIO[Msg]) tc.gaterFunc = func(core.Duty) bool { return true } - msg := &pbv1.ConsensusMsg{ - Msg: randomMsg(t), + msg := &pbv1.QBFTConsensusMsg{ + Msg: newRandomQBFTMsg(t), } test.mutate(msg, &tc) @@ -386,10 +391,10 @@ func TestComponent_handle(t *testing.T) { } } -func TestComponentHandle(t *testing.T) { +func TestQBFTConsensusHandle(t *testing.T) { tests := []struct { name string - msg *pbv1.ConsensusMsg + msg *pbv1.QBFTConsensusMsg errorMsg string peerID string }{ @@ -399,14 +404,14 @@ func TestComponentHandle(t *testing.T) { }, { name: "nil msg", - msg: &pbv1.ConsensusMsg{ + msg: &pbv1.QBFTConsensusMsg{ Msg: nil, }, errorMsg: "invalid consensus message", }, { name: "nil msg duty", - msg: &pbv1.ConsensusMsg{ + msg: &pbv1.QBFTConsensusMsg{ Msg: &pbv1.QBFTMsg{ Duty: nil, }, @@ -415,7 +420,7 @@ func TestComponentHandle(t *testing.T) { }, { name: "invalid consensus msg type", - msg: &pbv1.ConsensusMsg{ + msg: &pbv1.QBFTConsensusMsg{ Msg: &pbv1.QBFTMsg{ Duty: &pbv1.Duty{}, }, @@ -424,7 +429,7 @@ func TestComponentHandle(t *testing.T) { }, { name: "invalid msg duty type", - msg: &pbv1.ConsensusMsg{ + msg: &pbv1.QBFTConsensusMsg{ Msg: &pbv1.QBFTMsg{ Duty: &pbv1.Duty{}, Type: int64(qbft.MsgPrepare), @@ -434,7 +439,7 @@ func TestComponentHandle(t *testing.T) { }, { name: "invalid peer index", - msg: &pbv1.ConsensusMsg{ + msg: &pbv1.QBFTConsensusMsg{ Msg: &pbv1.QBFTMsg{ Round: 1, Duty: &pbv1.Duty{Type: int32(core.DutyProposer)}, @@ -449,7 +454,7 @@ func TestComponentHandle(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &Component{ + c := &Consensus{ gaterFunc: func(core.Duty) bool { return true }, } @@ -461,16 +466,18 @@ func TestComponentHandle(t *testing.T) { func TestInstanceIO_MaybeStart(t *testing.T) { t.Run("MaybeStart for new instance", func(t *testing.T) { - inst1 := newInstanceIO() + inst1 := utils.NewInstanceIO[Msg]() require.True(t, inst1.MaybeStart()) require.False(t, inst1.MaybeStart()) }) t.Run("MaybeStart after handle", func(t *testing.T) { - var c Component - c.deadliner = testDeadliner{} + var c Consensus + deadliner := coremocks.NewDeadliner(t) + deadliner.On("Add", mock.Anything).Return(true) + c.deadliner = deadliner c.gaterFunc = func(core.Duty) bool { return true } - c.mutable.instances = make(map[core.Duty]instanceIO) + c.mutable.instances = make(map[core.Duty]*utils.InstanceIO[Msg]) // Generate a p2p private key. p2pKey := testutil.GenerateInsecureK1Key(t, 0) @@ -478,8 +485,8 @@ func TestInstanceIO_MaybeStart(t *testing.T) { c.pubkeys[0] = p2pKey.PubKey() duty := core.Duty{Slot: 42, Type: 1} - msg := &pbv1.ConsensusMsg{ - Msg: randomMsg(t), + msg := &pbv1.QBFTConsensusMsg{ + Msg: newRandomQBFTMsg(t), } msg = signConsensusMsg(t, msg, p2pKey, duty) @@ -496,11 +503,13 @@ func TestInstanceIO_MaybeStart(t *testing.T) { t.Run("Call Propose after handle", func(t *testing.T) { ctx := context.Background() - var c Component - c.deadliner = testDeadliner{} + var c Consensus + deadliner := coremocks.NewDeadliner(t) + deadliner.On("Add", mock.Anything).Return(true) + c.deadliner = deadliner c.gaterFunc = func(core.Duty) bool { return true } - c.mutable.instances = make(map[core.Duty]instanceIO) - c.timerFunc = getTimerFunc() + c.mutable.instances = make(map[core.Duty]*utils.InstanceIO[Msg]) + c.timerFunc = utils.GetTimerFunc() // Generate a p2p private key pair. p2pKey := testutil.GenerateInsecureK1Key(t, 0) @@ -508,8 +517,8 @@ func TestInstanceIO_MaybeStart(t *testing.T) { c.pubkeys[0] = p2pKey.PubKey() duty := core.Duty{Slot: 42, Type: 1} - msg := &pbv1.ConsensusMsg{ - Msg: randomMsg(t), + msg := &pbv1.QBFTConsensusMsg{ + Msg: newRandomQBFTMsg(t), } msg = signConsensusMsg(t, msg, p2pKey, duty) @@ -530,20 +539,7 @@ func TestInstanceIO_MaybeStart(t *testing.T) { }) } -// testDeadliner is a mock deadliner implementation. -type testDeadliner struct { - deadlineChan chan core.Duty -} - -func (testDeadliner) Add(core.Duty) bool { - return true -} - -func (t testDeadliner) C() <-chan core.Duty { - return t.deadlineChan -} - -func signConsensusMsg(t *testing.T, msg *pbv1.ConsensusMsg, privKey *k1.PrivateKey, duty core.Duty) *pbv1.ConsensusMsg { +func signConsensusMsg(t *testing.T, msg *pbv1.QBFTConsensusMsg, privKey *k1.PrivateKey, duty core.Duty) *pbv1.QBFTConsensusMsg { t.Helper() msg.Msg.Duty.Type = int32(duty.Type) @@ -564,7 +560,7 @@ func signConsensusMsg(t *testing.T, msg *pbv1.ConsensusMsg, privKey *k1.PrivateK // construct a justification msg.Justification = []*pbv1.QBFTMsg{ - randomMsg(t), + newRandomQBFTMsg(t), } msg.Justification[0].PeerIdx = 0 diff --git a/core/consensus/component_test.go b/core/consensus/qbft/qbft_test.go similarity index 79% rename from core/consensus/component_test.go rename to core/consensus/qbft/qbft_test.go index 3568c309e..ef2aeba7c 100644 --- a/core/consensus/component_test.go +++ b/core/consensus/qbft/qbft_test.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus_test +package qbft_test import ( "context" @@ -14,20 +14,22 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/obolnetwork/charon/app/log" "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/cluster" "github.com/obolnetwork/charon/core" - "github.com/obolnetwork/charon/core/consensus" + "github.com/obolnetwork/charon/core/consensus/qbft" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" + coremocks "github.com/obolnetwork/charon/core/mocks" "github.com/obolnetwork/charon/eth2util/enr" "github.com/obolnetwork/charon/p2p" "github.com/obolnetwork/charon/testutil" ) -func TestComponent(t *testing.T) { +func TestQBFTConsensus(t *testing.T) { tests := []struct { name string threshold int @@ -57,19 +59,14 @@ func TestComponent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - testComponent(t, tt.threshold, tt.nodes) + testQBFTConsensus(t, tt.threshold, tt.nodes) }) } } -func TestIsSupportedProtocolName(t *testing.T) { - require.True(t, consensus.IsSupportedProtocolName("qbft")) - require.False(t, consensus.IsSupportedProtocolName("unreal")) -} - -// testComponent tests a consensus instance with size of threshold-of-nodes. +// testQBFTConsensus tests a consensus instance with size of threshold-of-nodes. // Note it only instantiates the minimum amount of peers, ie threshold. -func testComponent(t *testing.T, threshold, nodes int) { +func testQBFTConsensus(t *testing.T, threshold, nodes int) { t.Helper() seed := 0 random := rand.New(rand.NewSource(int64(seed))) @@ -79,7 +76,7 @@ func testComponent(t *testing.T, threshold, nodes int) { peers []p2p.Peer hosts []host.Host hostsInfo []peer.AddrInfo - components []*consensus.Component + components []*qbft.Consensus results = make(chan core.UnsignedDataSet, threshold) runErrs = make(chan error, threshold) sniffed = make(chan int, threshold) @@ -124,13 +121,16 @@ func testComponent(t *testing.T, threshold, nodes int) { gaterFunc := func(core.Duty) bool { return true } - c, err := consensus.New(hosts[i], new(p2p.Sender), peers, p2pkeys[i], testDeadliner{}, gaterFunc, sniffer) + deadliner := coremocks.NewDeadliner(t) + deadliner.On("Add", mock.Anything).Return(true) + deadliner.On("C").Return(nil) + c, err := qbft.NewConsensus(hosts[i], new(p2p.Sender), peers, p2pkeys[i], deadliner, gaterFunc, sniffer) require.NoError(t, err) c.Subscribe(func(_ context.Context, _ core.Duty, set core.UnsignedDataSet) error { results <- set return nil }) - c.Start(log.WithCtx(ctx, z.Int("node", i))) + c.Start(context.TODO()) components = append(components, c) } @@ -139,7 +139,7 @@ func testComponent(t *testing.T, threshold, nodes int) { // Start all components. for i, c := range components { - go func(ctx context.Context, i int, c *consensus.Component) { + go func(ctx context.Context, i int, c *qbft.Consensus) { runErrs <- c.Propose( log.WithCtx(ctx, z.Int("node", i), z.Str("peer", p2p.PeerName(hosts[i].ID()))), core.Duty{Type: core.DutyAttester, Slot: 1}, @@ -177,16 +177,3 @@ func testComponent(t *testing.T, threshold, nodes int) { require.NotZero(t, <-sniffed) } } - -// testDeadliner is a mock deadliner implementation. -type testDeadliner struct { - deadlineChan chan core.Duty -} - -func (testDeadliner) Add(core.Duty) bool { - return true -} - -func (t testDeadliner) C() <-chan core.Duty { - return t.deadlineChan -} diff --git a/core/consensus/sniffed_internal_test.go b/core/consensus/qbft/sniffed_internal_test.go similarity index 95% rename from core/consensus/sniffed_internal_test.go rename to core/consensus/qbft/sniffed_internal_test.go index 1364ae076..2fc7fb074 100644 --- a/core/consensus/sniffed_internal_test.go +++ b/core/consensus/qbft/sniffed_internal_test.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( "bytes" @@ -19,6 +19,7 @@ import ( "github.com/obolnetwork/charon/app/log" "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/utils" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" "github.com/obolnetwork/charon/core/qbft" ) @@ -76,7 +77,7 @@ func testSniffedInstance(ctx context.Context, t *testing.T, instance *pbv1.Sniff return nil }} - }, newIncreasingRoundTimer(), func(qcommit []qbft.Msg[core.Duty, [32]byte]) {}) + }, utils.NewIncreasingRoundTimer(), func(qcommit []qbft.Msg[core.Duty, [32]byte]) {}) recvBuffer := make(chan qbft.Msg[core.Duty, [32]byte], len(instance.GetMsgs())) diff --git a/core/consensus/qbft/sniffer.go b/core/consensus/qbft/sniffer.go new file mode 100644 index 000000000..12e76b75c --- /dev/null +++ b/core/consensus/qbft/sniffer.go @@ -0,0 +1,57 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package qbft + +import ( + "sync" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/obolnetwork/charon/core/consensus/protocols" + pbv1 "github.com/obolnetwork/charon/core/corepb/v1" +) + +// newSniffer returns a new sniffer. +func newSniffer(nodes, peerIdx int64) *sniffer { + return &sniffer{ + nodes: nodes, + peerIdx: peerIdx, + startedAt: time.Now(), + } +} + +// sniffer buffers consensus messages. +type sniffer struct { + nodes int64 + peerIdx int64 + startedAt time.Time + + mu sync.Mutex + msgs []*pbv1.SniffedConsensusMsg +} + +// Add adds a message to the sniffer buffer. +func (c *sniffer) Add(msg *pbv1.QBFTConsensusMsg) { + c.mu.Lock() + defer c.mu.Unlock() + + c.msgs = append(c.msgs, &pbv1.SniffedConsensusMsg{ + Timestamp: timestamppb.Now(), + Msg: msg, + }) +} + +// Instance returns the buffered messages as an instance. +func (c *sniffer) Instance() *pbv1.SniffedConsensusInstance { + c.mu.Lock() + defer c.mu.Unlock() + + return &pbv1.SniffedConsensusInstance{ + Nodes: c.nodes, + PeerIdx: c.peerIdx, + StartedAt: timestamppb.New(c.startedAt), + Msgs: c.msgs, + ProtocolId: protocols.QBFTv2ProtocolID, + } +} diff --git a/core/consensus/qbft/sniffer_internal_test.go b/core/consensus/qbft/sniffer_internal_test.go new file mode 100644 index 000000000..12abaca77 --- /dev/null +++ b/core/consensus/qbft/sniffer_internal_test.go @@ -0,0 +1,29 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package qbft + +import ( + "testing" + + "github.com/stretchr/testify/require" + + pbv1 "github.com/obolnetwork/charon/core/corepb/v1" +) + +func TestSniffer(t *testing.T) { + sniffer := newSniffer(3, 1) + + sniffer.Add(&pbv1.QBFTConsensusMsg{ + Msg: newRandomQBFTMsg(t), + }) + sniffer.Add(&pbv1.QBFTConsensusMsg{ + Msg: newRandomQBFTMsg(t), + }) + + instance := sniffer.Instance() + + require.EqualValues(t, 3, instance.GetNodes()) + require.EqualValues(t, 1, instance.GetPeerIdx()) + require.NotNil(t, instance.GetStartedAt()) + require.Len(t, instance.GetMsgs(), 2) +} diff --git a/core/consensus/strategysim_internal_test.go b/core/consensus/qbft/strategysim_internal_test.go similarity index 96% rename from core/consensus/strategysim_internal_test.go rename to core/consensus/qbft/strategysim_internal_test.go index a3e5acfe9..52fc0ce32 100644 --- a/core/consensus/strategysim_internal_test.go +++ b/core/consensus/qbft/strategysim_internal_test.go @@ -1,7 +1,7 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 //nolint:forbidigo // This is a test that prints to stdout. -package consensus +package qbft import ( "context" @@ -31,6 +31,7 @@ import ( "github.com/obolnetwork/charon/app/z" "github.com/obolnetwork/charon/cluster" "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/utils" pbv1 "github.com/obolnetwork/charon/core/corepb/v1" "github.com/obolnetwork/charon/core/qbft" ) @@ -53,7 +54,7 @@ const ( disabled = time.Hour * 999 ) -type roundTimerFunc func(clock clockwork.Clock) roundTimer +type roundTimerFunc func(clock clockwork.Clock) utils.RoundTimer func TestSimulatorOnce(t *testing.T) { syncer, _, _ := zap.Open("stderr") @@ -323,7 +324,7 @@ type ssConfig struct { latencyStdDev time.Duration latencyPerPeer map[int64]time.Duration startByPeer map[int64]time.Duration - roundTimerFunc func(clockwork.Clock) roundTimer + roundTimerFunc func(clockwork.Clock) utils.RoundTimer timeout time.Duration } @@ -457,7 +458,7 @@ func gosched() { } } -func newSimDefinition(nodes int, roundTimer roundTimer, +func newSimDefinition(nodes int, roundTimer utils.RoundTimer, decideCallback func(qcommit []qbft.Msg[core.Duty, [32]byte]), ) qbft.Definition[core.Duty, [32]byte] { quorum := qbft.Definition[int, int]{Nodes: nodes}.Quorum() @@ -498,7 +499,7 @@ func newSimDefinition(nodes int, roundTimer roundTimer, Nodes: nodes, // FIFOLimit caps the max buffered messages per peer. - FIFOLimit: recvBuffer, + FIFOLimit: utils.RecvBufferSize, } } @@ -631,11 +632,11 @@ func (i *transportInstance) Broadcast(_ context.Context, typ qbft.MsgType, // Transform justifications into protobufs var justMsgs []*pbv1.QBFTMsg for _, j := range justification { - impl, ok := j.(msg) + impl, ok := j.(Msg) if !ok { return errors.New("invalid justification") } - justMsgs = append(justMsgs, impl.msg) // Note nested justifications are ignored. + justMsgs = append(justMsgs, impl.Msg()) // Note nested justifications are ignored. values[impl.Value()] = dummy values[impl.PreparedValue()] = dummy } @@ -734,14 +735,14 @@ type incRoundTimer2 struct { clock clockwork.Clock } -func (t incRoundTimer2) Type() timerType { +func (t incRoundTimer2) Type() utils.TimerType { return "inc2" } func (t incRoundTimer2) Timer(round int64) (<-chan time.Time, func()) { - duration := incRoundStart + duration := utils.IncRoundStart for i := 1; i < int(round); i++ { - duration += incRoundStart + duration += utils.IncRoundStart } timer := t.clock.NewTimer(duration) @@ -749,7 +750,7 @@ func (t incRoundTimer2) Timer(round int64) (<-chan time.Time, func()) { return timer.Chan(), func() {} } -func randomConfigs(names []string, peers int, n int, timer func(clockwork.Clock) roundTimer, +func randomConfigs(names []string, peers int, n int, timer func(clockwork.Clock) utils.RoundTimer, stdDev []time.Duration, latencies []time.Duration, ) []ssConfig { random := rand.New(rand.NewSource(0)) @@ -902,17 +903,17 @@ func (t *testTimer) Timer(round int64) (<-chan time.Time, func()) { return timer.Chan(), func() {} } -func (t *testTimer) Type() timerType { +func (t *testTimer) Type() utils.TimerType { name := t.name if t.eager { name += "_eager" } - return timerType(name) + return utils.TimerType(name) } func newLinear(d time.Duration) roundTimerFunc { - return func(clock clockwork.Clock) roundTimer { + return func(clock clockwork.Clock) utils.RoundTimer { return &testTimer{ clock: clock, durationFunc: func(round int64) time.Duration { @@ -926,7 +927,7 @@ func newLinear(d time.Duration) roundTimerFunc { } func newExpDouble(d time.Duration) roundTimerFunc { - return func(clock clockwork.Clock) roundTimer { + return func(clock clockwork.Clock) utils.RoundTimer { return &testTimer{ clock: clock, durationFunc: func(round int64) time.Duration { @@ -941,7 +942,7 @@ func newExpDouble(d time.Duration) roundTimerFunc { } func newLinearDouble(d time.Duration) roundTimerFunc { - return func(clock clockwork.Clock) roundTimer { + return func(clock clockwork.Clock) utils.RoundTimer { return &testTimer{ clock: clock, durationFunc: func(round int64) time.Duration { @@ -955,12 +956,12 @@ func newLinearDouble(d time.Duration) roundTimerFunc { } } -func newInc(clock clockwork.Clock) roundTimer { - return &increasingRoundTimer{clock: clock} +func newInc(clock clockwork.Clock) utils.RoundTimer { + return utils.NewIncreasingRoundTimerWithClock(clock) } func newExp(d time.Duration) roundTimerFunc { - return func(clock clockwork.Clock) roundTimer { + return func(clock clockwork.Clock) utils.RoundTimer { return &testTimer{ clock: clock, durationFunc: func(round int64) time.Duration { diff --git a/core/consensus/testdata/TestDebugRoundChange_empty-1.golden b/core/consensus/qbft/testdata/TestDebugRoundChange_empty-1.golden similarity index 100% rename from core/consensus/testdata/TestDebugRoundChange_empty-1.golden rename to core/consensus/qbft/testdata/TestDebugRoundChange_empty-1.golden diff --git a/core/consensus/testdata/TestDebugRoundChange_empty-2.golden b/core/consensus/qbft/testdata/TestDebugRoundChange_empty-2.golden similarity index 100% rename from core/consensus/testdata/TestDebugRoundChange_empty-2.golden rename to core/consensus/qbft/testdata/TestDebugRoundChange_empty-2.golden diff --git a/core/consensus/testdata/TestDebugRoundChange_quorum.golden b/core/consensus/qbft/testdata/TestDebugRoundChange_quorum.golden similarity index 100% rename from core/consensus/testdata/TestDebugRoundChange_quorum.golden rename to core/consensus/qbft/testdata/TestDebugRoundChange_quorum.golden diff --git a/core/consensus/testdata/TestHashProto.golden b/core/consensus/qbft/testdata/TestHashProto.golden similarity index 100% rename from core/consensus/testdata/TestHashProto.golden rename to core/consensus/qbft/testdata/TestHashProto.golden diff --git a/core/consensus/transport.go b/core/consensus/qbft/transport.go similarity index 67% rename from core/consensus/transport.go rename to core/consensus/qbft/transport.go index 369825af7..7ed399619 100644 --- a/core/consensus/transport.go +++ b/core/consensus/qbft/transport.go @@ -1,16 +1,14 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package qbft import ( "context" "sync" - "time" k1 "github.com/decred/dcrd/dcrec/secp256k1/v4" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/obolnetwork/charon/app/errors" "github.com/obolnetwork/charon/core" @@ -18,12 +16,18 @@ import ( "github.com/obolnetwork/charon/core/qbft" ) +// broadcaster is an interface for broadcasting messages asynchronously. +type broadcaster interface { + Broadcast(ctx context.Context, msg *pbv1.QBFTConsensusMsg) error +} + // transport encapsulates receiving and broadcasting for a consensus instance/duty. type transport struct { // Immutable state - component *Component - recvBuffer chan qbft.Msg[core.Duty, [32]byte] // Instance inner receive buffer. - sniffer *sniffer + broadcaster broadcaster + privkey *k1.PrivateKey + recvBuffer chan qbft.Msg[core.Duty, [32]byte] // Instance inner receive buffer. + sniffer *sniffer // Mutable state valueMu sync.Mutex @@ -31,12 +35,26 @@ type transport struct { values map[[32]byte]*anypb.Any // maps any-wrapped proposed values to their hashes } +// newTransport creates a new qbftTransport. +func newTransport(broadcaster broadcaster, privkey *k1.PrivateKey, valueCh <-chan proto.Message, + recvBuffer chan qbft.Msg[core.Duty, [32]byte], sniffer *sniffer, +) *transport { + return &transport{ + broadcaster: broadcaster, + privkey: privkey, + recvBuffer: recvBuffer, + sniffer: sniffer, + valueCh: valueCh, + values: make(map[[32]byte]*anypb.Any), + } +} + // setValues caches the values and their hashes. -func (t *transport) setValues(msg msg) { +func (t *transport) setValues(msg Msg) { t.valueMu.Lock() defer t.valueMu.Unlock() - for k, v := range msg.values { + for k, v := range msg.Values() { t.values[k] = v } } @@ -82,12 +100,12 @@ func (t *transport) Broadcast(ctx context.Context, typ qbft.MsgType, duty core.D hashes = append(hashes, valueHash) hashes = append(hashes, pvHash) for _, just := range justification { - msg, ok := just.(msg) + msg, ok := just.(Msg) if !ok { return errors.New("invalid justification message") } - hashes = append(hashes, msg.valueHash) - hashes = append(hashes, msg.preparedValueHash) + hashes = append(hashes, msg.Value()) + hashes = append(hashes, msg.PreparedValue()) } // Get values by their hashes if not zero. @@ -107,7 +125,7 @@ func (t *transport) Broadcast(ctx context.Context, typ qbft.MsgType, duty core.D // Make the message msg, err := createMsg(typ, duty, peerIdx, round, valueHash, pr, - pvHash, values, justification, t.component.privkey) + pvHash, values, justification, t.privkey) if err != nil { return err } @@ -121,23 +139,11 @@ func (t *transport) Broadcast(ctx context.Context, typ qbft.MsgType, duty core.D } }() - for _, p := range t.component.peers { - if p.ID == t.component.tcpNode.ID() { - // Do not broadcast to self - continue - } - - err = t.component.sender.SendAsync(ctx, t.component.tcpNode, protocolID2, p.ID, msg.ToConsensusMsg()) - if err != nil { - return err - } - } - - return nil + return t.broadcaster.Broadcast(ctx, msg.ToConsensusMsg()) } // ProcessReceives processes received messages from the outer buffer until the context is closed. -func (t *transport) ProcessReceives(ctx context.Context, outerBuffer chan msg) { +func (t *transport) ProcessReceives(ctx context.Context, outerBuffer chan Msg) { for { select { case <-ctx.Done(): @@ -155,13 +161,23 @@ func (t *transport) ProcessReceives(ctx context.Context, outerBuffer chan msg) { } } +// SnifferInstance returns the current sniffed consensus instance. +func (t *transport) SnifferInstance() *pbv1.SniffedConsensusInstance { + return t.sniffer.Instance() +} + +// RecvBuffer returns the inner receive buffer. +func (t *transport) RecvBuffer() chan qbft.Msg[core.Duty, [32]byte] { + return t.recvBuffer +} + // createMsg returns a new message by converting the inputs into a protobuf // and wrapping that in a msg type. func createMsg(typ qbft.MsgType, duty core.Duty, peerIdx int64, round int64, vHash [32]byte, pr int64, pvHash [32]byte, values map[[32]byte]*anypb.Any, justification []qbft.Msg[core.Duty, [32]byte], privkey *k1.PrivateKey, -) (msg, error) { +) (Msg, error) { pbMsg := &pbv1.QBFTMsg{ Type: int64(typ), Duty: core.DutyToProto(duty), @@ -174,61 +190,18 @@ func createMsg(typ qbft.MsgType, duty core.Duty, pbMsg, err := signMsg(pbMsg, privkey) if err != nil { - return msg{}, err + return Msg{}, err } // Transform justifications into protobufs var justMsgs []*pbv1.QBFTMsg for _, j := range justification { - impl, ok := j.(msg) + impl, ok := j.(Msg) if !ok { - return msg{}, errors.New("invalid justification") + return Msg{}, errors.New("invalid justification") } - justMsgs = append(justMsgs, impl.msg) // Note nested justifications are ignored. + justMsgs = append(justMsgs, impl.Msg()) // Note nested justifications are ignored. } return newMsg(pbMsg, justMsgs, values) } - -// newSniffer returns a new sniffer. -func newSniffer(nodes, peerIdx int64) *sniffer { - return &sniffer{ - nodes: nodes, - peerIdx: peerIdx, - startedAt: time.Now(), - } -} - -// sniffer buffers consensus messages. -type sniffer struct { - nodes int64 - peerIdx int64 - startedAt time.Time - - mu sync.Mutex - msgs []*pbv1.SniffedConsensusMsg -} - -// Add adds a message to the sniffer buffer. -func (c *sniffer) Add(msg *pbv1.ConsensusMsg) { - c.mu.Lock() - defer c.mu.Unlock() - - c.msgs = append(c.msgs, &pbv1.SniffedConsensusMsg{ - Timestamp: timestamppb.Now(), - Msg: msg, - }) -} - -// Instance returns the buffered messages as an instance. -func (c *sniffer) Instance() *pbv1.SniffedConsensusInstance { - c.mu.Lock() - defer c.mu.Unlock() - - return &pbv1.SniffedConsensusInstance{ - Nodes: c.nodes, - PeerIdx: c.peerIdx, - StartedAt: timestamppb.New(c.startedAt), - Msgs: c.msgs, - } -} diff --git a/core/consensus/utils/instance_io.go b/core/consensus/utils/instance_io.go new file mode 100644 index 000000000..ea23f563e --- /dev/null +++ b/core/consensus/utils/instance_io.go @@ -0,0 +1,81 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package utils + +import ( + "time" + + "google.golang.org/protobuf/proto" + + "github.com/obolnetwork/charon/app/errors" +) + +const ( + RecvBufferSize = 100 // Allow buffering some initial messages when this node is late to start an instance. +) + +// NewInstanceIO returns a new instanceIO. +func NewInstanceIO[T any]() *InstanceIO[T] { + return &InstanceIO[T]{ + Participated: make(chan struct{}), + Proposed: make(chan struct{}), + Running: make(chan struct{}), + RecvBuffer: make(chan T, RecvBufferSize), + HashCh: make(chan [32]byte, 1), + ValueCh: make(chan proto.Message, 1), + ErrCh: make(chan error, 1), + DecidedAtCh: make(chan time.Time, 1), + } +} + +// InstanceIO defines the async input and output channels of a +// single consensus instance in the Component. +type InstanceIO[T any] struct { + Participated chan struct{} // Closed when Participate was called for this instance. + Proposed chan struct{} // Closed when Propose was called for this instance. + Running chan struct{} // Closed when runInstance was already called. + RecvBuffer chan T // Outer receive buffers. + HashCh chan [32]byte // Async input hash channel. + ValueCh chan proto.Message // Async input value channel. + ErrCh chan error // Async output error channel. + DecidedAtCh chan time.Time // Async output decided timestamp channel. +} + +// MarkParticipated marks the instance as participated. +// It returns an error if the instance was already marked as participated. +func (io *InstanceIO[T]) MarkParticipated() error { + select { + case <-io.Participated: + return errors.New("already participated") + default: + close(io.Participated) + } + + return nil +} + +// MarkProposed marks the instance as proposed. +// It returns an error if the instance was already marked as proposed. +func (io *InstanceIO[T]) MarkProposed() error { + select { + case <-io.Proposed: + return errors.New("already proposed") + default: + close(io.Proposed) + } + + return nil +} + +// MaybeStart returns true if the instance wasn't running and has been started by this call, +// otherwise it returns false if the instance was started in the past and is either running now or has completed. +func (io *InstanceIO[T]) MaybeStart() bool { + select { + case <-io.Running: + return false + default: + close(io.Running) + } + + return true +} diff --git a/core/consensus/utils/instance_io_test.go b/core/consensus/utils/instance_io_test.go new file mode 100644 index 000000000..34682d7e2 --- /dev/null +++ b/core/consensus/utils/instance_io_test.go @@ -0,0 +1,48 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package utils_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + cqbft "github.com/obolnetwork/charon/core/consensus/qbft" + "github.com/obolnetwork/charon/core/consensus/utils" +) + +func TestMarkParticipated(t *testing.T) { + io := utils.NewInstanceIO[cqbft.Msg]() + + // First call succeeds. + err := io.MarkParticipated() + require.NoError(t, err) + + // Second call fails. + err = io.MarkParticipated() + require.ErrorContains(t, err, "already participated") +} + +func TestMarkProposed(t *testing.T) { + io := utils.NewInstanceIO[cqbft.Msg]() + + // First call succeeds. + err := io.MarkProposed() + require.NoError(t, err) + + // Second call fails. + err = io.MarkProposed() + require.ErrorContains(t, err, "already proposed") +} + +func TestMaybeStart(t *testing.T) { + io := utils.NewInstanceIO[cqbft.Msg]() + + // First call succeeds. + ok := io.MaybeStart() + require.True(t, ok) + + // Second call fails. + ok = io.MaybeStart() + require.False(t, ok) +} diff --git a/core/consensus/roundtimer.go b/core/consensus/utils/roundtimer.go similarity index 63% rename from core/consensus/roundtimer.go rename to core/consensus/utils/roundtimer.go index c82e1555e..7a71228b6 100644 --- a/core/consensus/roundtimer.go +++ b/core/consensus/utils/roundtimer.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package utils import ( "strings" @@ -14,64 +14,69 @@ import ( ) const ( - incRoundStart = time.Millisecond * 750 - incRoundIncrease = time.Millisecond * 250 - linearRoundInc = time.Second + IncRoundStart = time.Millisecond * 750 + IncRoundIncrease = time.Millisecond * 250 + LinearRoundInc = time.Second ) -// timerFunc is a function that returns a round timer. -type timerFunc func(core.Duty) roundTimer +// TimerFunc is a function that returns a round timer. +type TimerFunc func(core.Duty) RoundTimer -// getTimerFunc returns a timer function based on the enabled features. -func getTimerFunc() timerFunc { +// GetTimerFunc returns a timer function based on the enabled features. +func GetTimerFunc() TimerFunc { if featureset.Enabled(featureset.EagerDoubleLinear) { - return func(core.Duty) roundTimer { - return newDoubleEagerLinearRoundTimer() + return func(core.Duty) RoundTimer { + return NewDoubleEagerLinearRoundTimer() } } // Default to increasing round timer. - return func(core.Duty) roundTimer { - return newIncreasingRoundTimer() + return func(core.Duty) RoundTimer { + return NewIncreasingRoundTimer() } } -// timerType is the type of round timer. -type timerType string +// TimerType is the type of round timer. +type TimerType string // Eager returns true if the timer type requires an eager start (before proposal values are present). -func (t timerType) Eager() bool { +func (t TimerType) Eager() bool { return strings.Contains(string(t), "eager") } const ( - timerIncreasing timerType = "inc" - timerEagerDoubleLinear timerType = "eager_dlinear" + TimerIncreasing TimerType = "inc" + TimerEagerDoubleLinear TimerType = "eager_dlinear" ) // increasingRoundTimeout returns the duration for a round that starts at incRoundStart in round 1 // and increases by incRoundIncrease for each subsequent round. func increasingRoundTimeout(round int64) time.Duration { - return incRoundStart + (time.Duration(round) * incRoundIncrease) + return IncRoundStart + (time.Duration(round) * IncRoundIncrease) } // increasingRoundTimeout returns linearRoundInc*round duration for a round. func linearRoundTimeout(round int64) time.Duration { - return time.Duration(round) * linearRoundInc + return time.Duration(round) * LinearRoundInc } -// roundTimer provides the duration for each QBFT round. -type roundTimer interface { +// RoundTimer provides the duration for each consensus round. +type RoundTimer interface { // Timer returns a channel that will be closed when the round expires and a stop function. Timer(round int64) (<-chan time.Time, func()) // Type returns the type of the round timerType. - Type() timerType + Type() TimerType } -// newTimeoutRoundTimer returns a new increasing round timerType. -func newIncreasingRoundTimer() roundTimer { +// NewTimeoutRoundTimer returns a new increasing round timer type. +func NewIncreasingRoundTimer() RoundTimer { + return NewIncreasingRoundTimerWithClock(clockwork.NewRealClock()) +} + +// NewIncreasingRoundTimerWithClock returns a new increasing round timer type with a custom clock. +func NewIncreasingRoundTimerWithClock(clock clockwork.Clock) RoundTimer { return &increasingRoundTimer{ - clock: clockwork.NewRealClock(), + clock: clock, } } @@ -80,8 +85,8 @@ type increasingRoundTimer struct { clock clockwork.Clock } -func (increasingRoundTimer) Type() timerType { - return timerIncreasing +func (increasingRoundTimer) Type() TimerType { + return TimerIncreasing } func (t increasingRoundTimer) Timer(round int64) (<-chan time.Time, func()) { @@ -89,10 +94,15 @@ func (t increasingRoundTimer) Timer(round int64) (<-chan time.Time, func()) { return timer.Chan(), func() { timer.Stop() } } -// doubleEagerLinearRoundTimer returns a new eager double linear round timerType. -func newDoubleEagerLinearRoundTimer() roundTimer { +// NewDoubleEagerLinearRoundTimer returns a new eager double linear round timer type. +func NewDoubleEagerLinearRoundTimer() RoundTimer { + return NewDoubleEagerLinearRoundTimerWithClock(clockwork.NewRealClock()) +} + +// NewDoubleEagerLinearRoundTimerWithClock returns a new eager double linear round timer type with a custom clock. +func NewDoubleEagerLinearRoundTimerWithClock(clock clockwork.Clock) RoundTimer { return &doubleEagerLinearRoundTimer{ - clock: clockwork.NewRealClock(), + clock: clock, firstDeadlines: make(map[int64]time.Time), } } @@ -118,8 +128,8 @@ type doubleEagerLinearRoundTimer struct { firstDeadlines map[int64]time.Time } -func (*doubleEagerLinearRoundTimer) Type() timerType { - return timerEagerDoubleLinear +func (*doubleEagerLinearRoundTimer) Type() TimerType { + return TimerEagerDoubleLinear } func (t *doubleEagerLinearRoundTimer) Timer(round int64) (<-chan time.Time, func()) { diff --git a/core/consensus/roundtimer_internal_test.go b/core/consensus/utils/roundtimer_test.go similarity index 74% rename from core/consensus/roundtimer_internal_test.go rename to core/consensus/utils/roundtimer_test.go index 9c73487f1..461c6377d 100644 --- a/core/consensus/roundtimer_internal_test.go +++ b/core/consensus/utils/roundtimer_test.go @@ -1,6 +1,6 @@ // Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 -package consensus +package utils_test import ( "testing" @@ -11,6 +11,7 @@ import ( "github.com/obolnetwork/charon/app/featureset" "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/utils" ) func TestIncreasingRoundTimer(t *testing.T) { @@ -38,8 +39,7 @@ func TestIncreasingRoundTimer(t *testing.T) { for _, tt := range tests { fakeClock := clockwork.NewFakeClock() - timer := newIncreasingRoundTimer().(*increasingRoundTimer) - timer.clock = fakeClock + timer := utils.NewIncreasingRoundTimerWithClock(fakeClock) t.Run(tt.name, func(t *testing.T) { // Start the timerType @@ -63,8 +63,7 @@ func TestIncreasingRoundTimer(t *testing.T) { func TestDoubleEagerLinearRoundTimer(t *testing.T) { fakeClock := clockwork.NewFakeClock() - timer := newDoubleEagerLinearRoundTimer().(*doubleEagerLinearRoundTimer) - timer.clock = fakeClock + timer := utils.NewDoubleEagerLinearRoundTimerWithClock(fakeClock) require.True(t, timer.Type().Eager()) @@ -113,16 +112,16 @@ func TestDoubleEagerLinearRoundTimer(t *testing.T) { } func TestGetTimerFunc(t *testing.T) { - timerFunc := getTimerFunc() - require.Equal(t, timerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(0)).Type()) - require.Equal(t, timerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(1)).Type()) - require.Equal(t, timerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(2)).Type()) + timerFunc := utils.GetTimerFunc() + require.Equal(t, utils.TimerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(0)).Type()) + require.Equal(t, utils.TimerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(1)).Type()) + require.Equal(t, utils.TimerEagerDoubleLinear, timerFunc(core.NewAttesterDuty(2)).Type()) featureset.DisableForT(t, featureset.EagerDoubleLinear) - timerFunc = getTimerFunc() + timerFunc = utils.GetTimerFunc() - require.Equal(t, timerIncreasing, timerFunc(core.NewAttesterDuty(0)).Type()) - require.Equal(t, timerIncreasing, timerFunc(core.NewAttesterDuty(1)).Type()) - require.Equal(t, timerIncreasing, timerFunc(core.NewAttesterDuty(2)).Type()) + require.Equal(t, utils.TimerIncreasing, timerFunc(core.NewAttesterDuty(0)).Type()) + require.Equal(t, utils.TimerIncreasing, timerFunc(core.NewAttesterDuty(1)).Type()) + require.Equal(t, utils.TimerIncreasing, timerFunc(core.NewAttesterDuty(2)).Type()) } diff --git a/core/consensus/wrapper.go b/core/consensus/wrapper.go new file mode 100644 index 000000000..f5aa66a5e --- /dev/null +++ b/core/consensus/wrapper.go @@ -0,0 +1,69 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package consensus + +import ( + "context" + "sync" + + "github.com/libp2p/go-libp2p/core/protocol" + + "github.com/obolnetwork/charon/core" +) + +type consensusWrapper struct { + lock sync.RWMutex + impl core.Consensus +} + +var _ core.Consensus = (*consensusWrapper)(nil) + +// newConsensusWrapper wraps a core.Consensus implementation. +func newConsensusWrapper(impl core.Consensus) *consensusWrapper { + return &consensusWrapper{ + impl: impl, + } +} + +// SetImpl sets the core.Consensus implementation. +func (w *consensusWrapper) SetImpl(impl core.Consensus) { + w.lock.Lock() + defer w.lock.Unlock() + + w.impl = impl +} + +func (w *consensusWrapper) ProtocolID() protocol.ID { + w.lock.RLock() + defer w.lock.RUnlock() + + return w.impl.ProtocolID() +} + +func (w *consensusWrapper) Start(ctx context.Context) { + w.lock.RLock() + defer w.lock.RUnlock() + + w.impl.Start(ctx) +} + +func (w *consensusWrapper) Participate(ctx context.Context, duty core.Duty) error { + w.lock.RLock() + defer w.lock.RUnlock() + + return w.impl.Participate(ctx, duty) +} + +func (w *consensusWrapper) Propose(ctx context.Context, duty core.Duty, dataSet core.UnsignedDataSet) error { + w.lock.RLock() + defer w.lock.RUnlock() + + return w.impl.Propose(ctx, duty, dataSet) +} + +func (w *consensusWrapper) Subscribe(fn func(context.Context, core.Duty, core.UnsignedDataSet) error) { + w.lock.RLock() + defer w.lock.RUnlock() + + w.impl.Subscribe(fn) +} diff --git a/core/consensus/wrapper_internal_test.go b/core/consensus/wrapper_internal_test.go new file mode 100644 index 000000000..fc2c555b1 --- /dev/null +++ b/core/consensus/wrapper_internal_test.go @@ -0,0 +1,53 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package consensus + +import ( + "context" + "testing" + + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/core" + "github.com/obolnetwork/charon/core/consensus/protocols" + "github.com/obolnetwork/charon/core/mocks" +) + +func TestNewConsensusWrapper(t *testing.T) { + ctx := context.Background() + randaoDuty := core.NewRandaoDuty(123) + dataSet := core.UnsignedDataSet{} + + impl := mocks.NewConsensus(t) + impl.On("ProtocolID").Return(protocol.ID(protocols.QBFTv2ProtocolID)) + impl.On("Participate", ctx, randaoDuty).Return(nil) + impl.On("Propose", ctx, randaoDuty, dataSet).Return(nil) + impl.On("Subscribe", mock.Anything).Return() + impl.On("Start", mock.Anything).Return() + + wrapped := newConsensusWrapper(impl) + require.NotNil(t, wrapped) + + require.EqualValues(t, protocols.QBFTv2ProtocolID, wrapped.ProtocolID()) + + err := wrapped.Participate(ctx, randaoDuty) + require.NoError(t, err) + + err = wrapped.Propose(ctx, randaoDuty, dataSet) + require.NoError(t, err) + + wrapped.Subscribe(func(ctx context.Context, d core.Duty, uds core.UnsignedDataSet) error { + return nil + }) + + wrapped.Start(ctx) + + impl2 := mocks.NewConsensus(t) + impl2.On("ProtocolID").Return(protocol.ID("foobar")) + + wrapped.SetImpl(impl2) + + require.EqualValues(t, "foobar", wrapped.ProtocolID()) +} diff --git a/core/corepb/v1/consensus.pb.go b/core/corepb/v1/consensus.pb.go index fb70b6c67..bc83fd472 100644 --- a/core/corepb/v1/consensus.pb.go +++ b/core/corepb/v1/consensus.pb.go @@ -123,7 +123,7 @@ func (x *QBFTMsg) GetPreparedValueHash() []byte { return nil } -type ConsensusMsg struct { +type QBFTConsensusMsg struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -133,20 +133,20 @@ type ConsensusMsg struct { Values []*anypb.Any `protobuf:"bytes,3,rep,name=values,proto3" json:"values,omitempty"` // values of the hashes in the messages } -func (x *ConsensusMsg) Reset() { - *x = ConsensusMsg{} +func (x *QBFTConsensusMsg) Reset() { + *x = QBFTConsensusMsg{} mi := &file_core_corepb_v1_consensus_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ConsensusMsg) String() string { +func (x *QBFTConsensusMsg) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ConsensusMsg) ProtoMessage() {} +func (*QBFTConsensusMsg) ProtoMessage() {} -func (x *ConsensusMsg) ProtoReflect() protoreflect.Message { +func (x *QBFTConsensusMsg) ProtoReflect() protoreflect.Message { mi := &file_core_corepb_v1_consensus_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -158,26 +158,26 @@ func (x *ConsensusMsg) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ConsensusMsg.ProtoReflect.Descriptor instead. -func (*ConsensusMsg) Descriptor() ([]byte, []int) { +// Deprecated: Use QBFTConsensusMsg.ProtoReflect.Descriptor instead. +func (*QBFTConsensusMsg) Descriptor() ([]byte, []int) { return file_core_corepb_v1_consensus_proto_rawDescGZIP(), []int{1} } -func (x *ConsensusMsg) GetMsg() *QBFTMsg { +func (x *QBFTConsensusMsg) GetMsg() *QBFTMsg { if x != nil { return x.Msg } return nil } -func (x *ConsensusMsg) GetJustification() []*QBFTMsg { +func (x *QBFTConsensusMsg) GetJustification() []*QBFTMsg { if x != nil { return x.Justification } return nil } -func (x *ConsensusMsg) GetValues() []*anypb.Any { +func (x *QBFTConsensusMsg) GetValues() []*anypb.Any { if x != nil { return x.Values } @@ -190,7 +190,7 @@ type SniffedConsensusMsg struct { unknownFields protoimpl.UnknownFields Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Msg *ConsensusMsg `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` + Msg *QBFTConsensusMsg `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` // Other consensus protocol messages can be added here } func (x *SniffedConsensusMsg) Reset() { @@ -230,7 +230,7 @@ func (x *SniffedConsensusMsg) GetTimestamp() *timestamppb.Timestamp { return nil } -func (x *SniffedConsensusMsg) GetMsg() *ConsensusMsg { +func (x *SniffedConsensusMsg) GetMsg() *QBFTConsensusMsg { if x != nil { return x.Msg } @@ -242,10 +242,11 @@ type SniffedConsensusInstance struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - StartedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` - Nodes int64 `protobuf:"varint,2,opt,name=nodes,proto3" json:"nodes,omitempty"` - PeerIdx int64 `protobuf:"varint,3,opt,name=peer_idx,json=peerIdx,proto3" json:"peer_idx,omitempty"` - Msgs []*SniffedConsensusMsg `protobuf:"bytes,4,rep,name=msgs,proto3" json:"msgs,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + Nodes int64 `protobuf:"varint,2,opt,name=nodes,proto3" json:"nodes,omitempty"` + PeerIdx int64 `protobuf:"varint,3,opt,name=peer_idx,json=peerIdx,proto3" json:"peer_idx,omitempty"` + Msgs []*SniffedConsensusMsg `protobuf:"bytes,4,rep,name=msgs,proto3" json:"msgs,omitempty"` + ProtocolId string `protobuf:"bytes,5,opt,name=protocol_id,json=protocolId,proto3" json:"protocol_id,omitempty"` } func (x *SniffedConsensusInstance) Reset() { @@ -306,6 +307,13 @@ func (x *SniffedConsensusInstance) GetMsgs() []*SniffedConsensusMsg { return nil } +func (x *SniffedConsensusInstance) GetProtocolId() string { + if x != nil { + return x.ProtocolId + } + return "" +} + type SniffedConsensusInstances struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -388,50 +396,52 @@ var file_core_corepb_v1_consensus_proto_rawDesc = []byte{ 0x65, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x61, 0x73, 0x68, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x07, - 0x10, 0x08, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x22, 0xa6, - 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x4d, 0x73, 0x67, 0x12, - 0x29, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x42, - 0x46, 0x54, 0x4d, 0x73, 0x67, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, 0x3d, 0x0a, 0x0d, 0x6a, 0x75, - 0x73, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, - 0x76, 0x31, 0x2e, 0x51, 0x42, 0x46, 0x54, 0x4d, 0x73, 0x67, 0x52, 0x0d, 0x6a, 0x75, 0x73, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x06, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, - 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x7f, 0x0a, 0x13, 0x53, 0x6e, 0x69, 0x66, 0x66, - 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x4d, 0x73, 0x67, 0x12, 0x38, - 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x2e, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, - 0x4d, 0x73, 0x67, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xbf, 0x01, 0x0a, 0x18, 0x53, 0x6e, 0x69, - 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, - 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x70, 0x65, 0x65, 0x72, 0x49, 0x64, - 0x78, 0x12, 0x37, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, - 0x73, 0x4d, 0x73, 0x67, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x7e, 0x0a, 0x19, 0x53, 0x6e, - 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x46, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x69, 0x66, - 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, - 0x19, 0x0a, 0x08, 0x67, 0x69, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x67, 0x69, 0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x62, 0x6f, 0x6c, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x63, 0x68, 0x61, 0x72, 0x6f, 0x6e, 0x2f, 0x63, 0x6f, 0x72, 0x65, - 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x10, 0x08, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x22, 0xaa, + 0x01, 0x0a, 0x10, 0x51, 0x42, 0x46, 0x54, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, + 0x4d, 0x73, 0x67, 0x12, 0x29, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, + 0x31, 0x2e, 0x51, 0x42, 0x46, 0x54, 0x4d, 0x73, 0x67, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, 0x3d, + 0x0a, 0x0d, 0x6a, 0x75, 0x73, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x42, 0x46, 0x54, 0x4d, 0x73, 0x67, 0x52, 0x0d, + 0x6a, 0x75, 0x73, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, + 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x41, 0x6e, 0x79, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x83, 0x01, 0x0a, 0x13, + 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, + 0x4d, 0x73, 0x67, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x32, 0x0a, + 0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x42, 0x46, 0x54, + 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x4d, 0x73, 0x67, 0x52, 0x03, 0x6d, 0x73, + 0x67, 0x22, 0xe0, 0x01, 0x0a, 0x18, 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, + 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x39, + 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x6f, 0x64, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x07, 0x70, 0x65, 0x65, 0x72, 0x49, 0x64, 0x78, 0x12, 0x37, 0x0a, 0x04, 0x6d, 0x73, + 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, + 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x4d, 0x73, 0x67, 0x52, 0x04, 0x6d, + 0x73, 0x67, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x49, 0x64, 0x22, 0x7e, 0x0a, 0x19, 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, + 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x73, 0x12, 0x46, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x70, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x69, 0x66, 0x66, 0x65, 0x64, 0x43, 0x6f, 0x6e, + 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x09, + 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x67, 0x69, 0x74, + 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x69, 0x74, + 0x48, 0x61, 0x73, 0x68, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x62, 0x6f, 0x6c, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x63, + 0x68, 0x61, 0x72, 0x6f, 0x6e, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x70, + 0x62, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -449,7 +459,7 @@ func file_core_corepb_v1_consensus_proto_rawDescGZIP() []byte { var file_core_corepb_v1_consensus_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_core_corepb_v1_consensus_proto_goTypes = []any{ (*QBFTMsg)(nil), // 0: core.corepb.v1.QBFTMsg - (*ConsensusMsg)(nil), // 1: core.corepb.v1.ConsensusMsg + (*QBFTConsensusMsg)(nil), // 1: core.corepb.v1.QBFTConsensusMsg (*SniffedConsensusMsg)(nil), // 2: core.corepb.v1.SniffedConsensusMsg (*SniffedConsensusInstance)(nil), // 3: core.corepb.v1.SniffedConsensusInstance (*SniffedConsensusInstances)(nil), // 4: core.corepb.v1.SniffedConsensusInstances @@ -459,11 +469,11 @@ var file_core_corepb_v1_consensus_proto_goTypes = []any{ } var file_core_corepb_v1_consensus_proto_depIdxs = []int32{ 5, // 0: core.corepb.v1.QBFTMsg.duty:type_name -> core.corepb.v1.Duty - 0, // 1: core.corepb.v1.ConsensusMsg.msg:type_name -> core.corepb.v1.QBFTMsg - 0, // 2: core.corepb.v1.ConsensusMsg.justification:type_name -> core.corepb.v1.QBFTMsg - 6, // 3: core.corepb.v1.ConsensusMsg.values:type_name -> google.protobuf.Any + 0, // 1: core.corepb.v1.QBFTConsensusMsg.msg:type_name -> core.corepb.v1.QBFTMsg + 0, // 2: core.corepb.v1.QBFTConsensusMsg.justification:type_name -> core.corepb.v1.QBFTMsg + 6, // 3: core.corepb.v1.QBFTConsensusMsg.values:type_name -> google.protobuf.Any 7, // 4: core.corepb.v1.SniffedConsensusMsg.timestamp:type_name -> google.protobuf.Timestamp - 1, // 5: core.corepb.v1.SniffedConsensusMsg.msg:type_name -> core.corepb.v1.ConsensusMsg + 1, // 5: core.corepb.v1.SniffedConsensusMsg.msg:type_name -> core.corepb.v1.QBFTConsensusMsg 7, // 6: core.corepb.v1.SniffedConsensusInstance.started_at:type_name -> google.protobuf.Timestamp 2, // 7: core.corepb.v1.SniffedConsensusInstance.msgs:type_name -> core.corepb.v1.SniffedConsensusMsg 3, // 8: core.corepb.v1.SniffedConsensusInstances.instances:type_name -> core.corepb.v1.SniffedConsensusInstance diff --git a/core/corepb/v1/consensus.proto b/core/corepb/v1/consensus.proto index 289c5d98c..b20995c5b 100644 --- a/core/corepb/v1/consensus.proto +++ b/core/corepb/v1/consensus.proto @@ -13,9 +13,9 @@ message QBFTMsg { core.corepb.v1.Duty duty = 2; int64 peer_idx = 3; int64 round = 4; - reserved 5 ; + reserved 5; int64 prepared_round = 6; - reserved 7 ; + reserved 7; bytes signature = 8; reserved 9; reserved 10; @@ -23,7 +23,7 @@ message QBFTMsg { bytes prepared_value_hash = 12; } -message ConsensusMsg { +message QBFTConsensusMsg { QBFTMsg msg = 1; // msg is the message that we send repeated QBFTMsg justification = 2; // justification is the justifications from others for the message repeated google.protobuf.Any values = 3; // values of the hashes in the messages @@ -31,17 +31,19 @@ message ConsensusMsg { message SniffedConsensusMsg { google.protobuf.Timestamp timestamp = 1; - ConsensusMsg msg = 2; + QBFTConsensusMsg msg = 2; + // Other consensus protocol messages can be added here } message SniffedConsensusInstance { - google.protobuf.Timestamp started_at = 1; - int64 nodes = 2; - int64 peer_idx = 3; - repeated SniffedConsensusMsg msgs = 4; + google.protobuf.Timestamp started_at = 1; + int64 nodes = 2; + int64 peer_idx = 3; + repeated SniffedConsensusMsg msgs = 4; + string protocol_id = 5; } message SniffedConsensusInstances { repeated SniffedConsensusInstance instances = 1; - string git_hash = 2; + string git_hash = 2; } diff --git a/core/deadline.go b/core/deadline.go index ef357dde4..db53dece3 100644 --- a/core/deadline.go +++ b/core/deadline.go @@ -16,6 +16,8 @@ import ( "github.com/obolnetwork/charon/app/z" ) +//go:generate mockery --name=Deadliner --output=mocks --outpkg=mocks --case=underscore + // lateFactor defines the number of slots duties may be late. // See https://pintail.xyz/posts/modelling-the-impact-of-altair/#proposer-and-delay-rewards. const lateFactor = 5 @@ -23,6 +25,9 @@ const lateFactor = 5 // lateMin defines the minimum absolute value of the lateFactor. const lateMin = time.Second * 30 //nolint:revive // Min suffix is minimum not minute. +// DeadlineFunc is a function that returns the deadline for a duty. +type DeadlineFunc func(Duty) (time.Time, bool) + // Deadliner provides duty Deadline functionality. The C method isn’t thread safe and // may only be used by a single goroutine. So, multiple instances are required // for different components and use cases. @@ -53,7 +58,7 @@ type deadliner struct { } // NewDeadlinerForT returns a Deadline for use in tests. -func NewDeadlinerForT(ctx context.Context, t *testing.T, deadlineFunc func(Duty) (time.Time, bool), clock clockwork.Clock) Deadliner { +func NewDeadlinerForT(ctx context.Context, t *testing.T, deadlineFunc DeadlineFunc, clock clockwork.Clock) Deadliner { t.Helper() return newDeadliner(ctx, "test", deadlineFunc, clock) @@ -63,12 +68,12 @@ func NewDeadlinerForT(ctx context.Context, t *testing.T, deadlineFunc func(Duty) // // It also starts a goroutine which is responsible for reading and storing duties, // and sending the deadlined duty to receiver's deadlineChan until the context is closed. -func NewDeadliner(ctx context.Context, label string, deadlineFunc func(Duty) (time.Time, bool)) Deadliner { +func NewDeadliner(ctx context.Context, label string, deadlineFunc DeadlineFunc) Deadliner { return newDeadliner(ctx, label, deadlineFunc, clockwork.NewRealClock()) } // newDeadliner returns a new Deadliner, this is for internal use only. -func newDeadliner(ctx context.Context, label string, deadlineFunc func(Duty) (time.Time, bool), clock clockwork.Clock) Deadliner { +func newDeadliner(ctx context.Context, label string, deadlineFunc DeadlineFunc, clock clockwork.Clock) Deadliner { // outputBuffer big enough to support all duty types, which can expire at the same time // while external consumer is synchronously adding duties (so not reading output). const outputBuffer = 10 @@ -86,7 +91,7 @@ func newDeadliner(ctx context.Context, label string, deadlineFunc func(Duty) (ti return d } -func (d *deadliner) run(ctx context.Context, deadlineFunc func(Duty) (time.Time, bool)) { +func (d *deadliner) run(ctx context.Context, deadlineFunc DeadlineFunc) { duties := make(map[Duty]bool) currDuty, currDeadline := getCurrDuty(duties, deadlineFunc) currTimer := d.clock.NewTimer(currDeadline.Sub(d.clock.Now())) @@ -172,7 +177,7 @@ func (d *deadliner) C() <-chan Duty { } // getCurrDuty gets the duty to process next along-with the duty deadline. It selects duty with the latest deadline. -func getCurrDuty(duties map[Duty]bool, deadlineFunc func(duty Duty) (time.Time, bool)) (Duty, time.Time) { +func getCurrDuty(duties map[Duty]bool, deadlineFunc DeadlineFunc) (Duty, time.Time) { var currDuty Duty currDeadline := time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC) @@ -193,7 +198,7 @@ func getCurrDuty(duties map[Duty]bool, deadlineFunc func(duty Duty) (time.Time, } // NewDutyDeadlineFunc returns the function that provides duty deadlines or false if the duty never deadlines. -func NewDutyDeadlineFunc(ctx context.Context, eth2Cl eth2wrap.Client) (func(Duty) (time.Time, bool), error) { +func NewDutyDeadlineFunc(ctx context.Context, eth2Cl eth2wrap.Client) (DeadlineFunc, error) { genesis, err := eth2Cl.GenesisTime(ctx) if err != nil { return nil, err diff --git a/core/interfaces.go b/core/interfaces.go index 786876124..73a74a687 100644 --- a/core/interfaces.go +++ b/core/interfaces.go @@ -8,8 +8,11 @@ import ( eth2api "github.com/attestantio/go-eth2-client/api" "github.com/attestantio/go-eth2-client/spec/altair" eth2p0 "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/libp2p/go-libp2p/core/protocol" ) +//go:generate mockery --name=Consensus --output=mocks --outpkg=mocks --case=underscore + // Scheduler triggers the start of a duty workflow. type Scheduler interface { // SubscribeDuties subscribes a callback function for triggered duties. @@ -65,8 +68,20 @@ type DutyDB interface { AwaitSyncContribution(ctx context.Context, slot, subcommIdx uint64, beaconBlockRoot eth2p0.Root) (*altair.SyncCommitteeContribution, error) } +// P2PProtocol defines an arbitrary libp2p protocol. +type P2PProtocol interface { + // ProtocolID returns the protocol ID. + ProtocolID() protocol.ID + + // Start registers libp2p handler and runs internal routines until the context is cancelled. + // The protocol must be unregistered when the context is cancelled. + Start(context.Context) +} + // Consensus comes to consensus on proposed duty data. type Consensus interface { + P2PProtocol + // Participate run the duty's consensus instance without a proposed value (if Propose not called yet). Participate(context.Context, Duty) error @@ -77,6 +92,28 @@ type Consensus interface { Subscribe(func(context.Context, Duty, UnsignedDataSet) error) } +// ConsensusController manages consensus instances. +type ConsensusController interface { + // Start starts the consensus controller lifecycle. + // The function is not thread safe, must be called once. + Start(context.Context) + + // DefaultConsensus returns the default consensus instance. + // The default consensus must be QBFT v2.0, since it is supported by all charon versions. + // It is used for Priority protocol as well as the fallback protocol when no other protocol is selected. + // Multiple calls to DefaultConsensus must return the same instance. + DefaultConsensus() Consensus + + // CurrentConsensus returns the currently selected consensus instance. + // The instance is selected by the Priority protocol and can be changed by SetCurrentConsensusForProtocol(). + // Before SetCurrentConsensusForProtocol() is called, CurrentConsensus() points to DefaultConsensus(). + CurrentConsensus() Consensus + + // SetCurrentConsensusForProtocol handles Priority protocol outcome and changes the CurrentConsensus() accordingly. + // The function is not thread safe. + SetCurrentConsensusForProtocol(context.Context, protocol.ID) error +} + // ValidatorAPI provides a beacon node API to validator clients. It serves duty data from the DutyDB and stores partial signed data in the ParSigDB. type ValidatorAPI interface { // RegisterAwaitProposal registers a function to query unsigned beacon block proposals by providing the slot. diff --git a/core/mocks/consensus.go b/core/mocks/consensus.go new file mode 100644 index 000000000..f5d282905 --- /dev/null +++ b/core/mocks/consensus.go @@ -0,0 +1,97 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +// Code generated by mockery v2.42.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + core "github.com/obolnetwork/charon/core" + mock "github.com/stretchr/testify/mock" + + protocol "github.com/libp2p/go-libp2p/core/protocol" +) + +// Consensus is an autogenerated mock type for the Consensus type +type Consensus struct { + mock.Mock +} + +// Participate provides a mock function with given fields: _a0, _a1 +func (_m *Consensus) Participate(_a0 context.Context, _a1 core.Duty) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Participate") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, core.Duty) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Propose provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Consensus) Propose(_a0 context.Context, _a1 core.Duty, _a2 core.UnsignedDataSet) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for Propose") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, core.Duty, core.UnsignedDataSet) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProtocolID provides a mock function with given fields: +func (_m *Consensus) ProtocolID() protocol.ID { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ProtocolID") + } + + var r0 protocol.ID + if rf, ok := ret.Get(0).(func() protocol.ID); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(protocol.ID) + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *Consensus) Start(_a0 context.Context) { + _m.Called(_a0) +} + +// Subscribe provides a mock function with given fields: _a0 +func (_m *Consensus) Subscribe(_a0 func(context.Context, core.Duty, core.UnsignedDataSet) error) { + _m.Called(_a0) +} + +// NewConsensus creates a new instance of Consensus. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConsensus(t interface { + mock.TestingT + Cleanup(func()) +}) *Consensus { + mock := &Consensus{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/mocks/deadliner.go b/core/mocks/deadliner.go new file mode 100644 index 000000000..5c249064b --- /dev/null +++ b/core/mocks/deadliner.go @@ -0,0 +1,67 @@ +// Copyright © 2022-2024 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +// Code generated by mockery v2.42.1. DO NOT EDIT. + +package mocks + +import ( + core "github.com/obolnetwork/charon/core" + mock "github.com/stretchr/testify/mock" +) + +// Deadliner is an autogenerated mock type for the Deadliner type +type Deadliner struct { + mock.Mock +} + +// Add provides a mock function with given fields: duty +func (_m *Deadliner) Add(duty core.Duty) bool { + ret := _m.Called(duty) + + if len(ret) == 0 { + panic("no return value specified for Add") + } + + var r0 bool + if rf, ok := ret.Get(0).(func(core.Duty) bool); ok { + r0 = rf(duty) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// C provides a mock function with given fields: +func (_m *Deadliner) C() <-chan core.Duty { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for C") + } + + var r0 <-chan core.Duty + if rf, ok := ret.Get(0).(func() <-chan core.Duty); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan core.Duty) + } + } + + return r0 +} + +// NewDeadliner creates a new instance of Deadliner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewDeadliner(t interface { + mock.TestingT + Cleanup(func()) +}) *Deadliner { + mock := &Deadliner{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/docs/README.md b/docs/README.md index 8bc6bddde..30a414217 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,3 +11,4 @@ This page acts as an index for the charon (pronounced 'kharon') markdown documen - [Go Guidelines](goguidelines.md): Guidelines and principals relating to go development - [Contributing](contributing.md): How to contribute to charon; githooks, PR templates, etc. - [Distributed Key Generation](dkg.md): How charon can create distributed validator key shares remotely from a cluster-definition file. +- [Consensus](consensus.md): How charon handles various consensus protocols. diff --git a/docs/configuration.md b/docs/configuration.md index 4d30b14ef..45dd9777b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -153,6 +153,7 @@ Flags: --beacon-node-submit-timeout duration Timeout for the submission-related HTTP requests Charon makes to the configured beacon nodes. (default 2s) --beacon-node-timeout duration Timeout for the HTTP requests Charon makes to the configured beacon nodes. (default 2s) --builder-api Enables the builder api. Will only produce builder blocks. Builder API must also be enabled on the validator client. Beacon node must be connected to a builder-relay to access the builder network. + --consensus-protocol string Preferred consensus protocol name for the node. Selected automatically when not specified. --debug-address string Listening address (ip and port) for the pprof and QBFT debug API. It is not enabled by default. --feature-set string Minimum feature set to enable by default: alpha, beta, or stable. Warning: modify at own risk. (default "stable") --feature-set-disable strings Comma-separated list of features to disable, overriding the default minimum feature set. diff --git a/docs/consensus.md b/docs/consensus.md new file mode 100644 index 000000000..dfdc08025 --- /dev/null +++ b/docs/consensus.md @@ -0,0 +1,81 @@ +# Consensus + +This document describes how Charon handles various consensus protocols. + +## Overview + +Historically, Charon has supported the single consensus protocol QBFT v2.0. +However, now the consensus layer has a pluggable interface that allows running different consensus protocols as long as they are available and accepted by the majority of the cluster. Moreover, the cluster can run multiple consensus protocols at the same time, e.g., for different purposes. + +## Consensus Protocol Selection + +The cluster nodes must agree on the preferred consensus protocol to use, otherwise, the entire consensus will fail. +Each node, depending on its configuration and software version, may prefer one or more consensus protocols in a specific order of precedence. +Charon runs a special protocol called Priority, which achieves consensus on the preferred consensus protocol to use. +Under the hood, this protocol uses the existing QBFT v2.0 algorithm that has been present since v0.19 and must not be deprecated. +This way, the existing QBFT v2.0 remains present for all future Charon versions to serve two purposes: running the Priority protocol and being a fallback protocol if no other protocol is selected. + +### Priority Protocol Input and Output + +The input to the Priority protocol is a list of protocols defined in order of precedence, e.g.: + +```json +[ + "/charon/consensus/hotstuff/1.0.0", // Highest precedence + "/charon/consensus/abft/2.0.0", + "/charon/consensus/abft/1.0.0", + "/charon/consensus/qbft/2.0.0", // Lowest precedence and the fallback since it is always present +] +``` + +The output of the Priority protocol is the common "subset" of all inputs, respecting the initial order of precedence, e.g.: + +```json +[ + "/charon/consensus/abft/1.0.0", // This means the majority of nodes have this protocol available + "/charon/consensus/qbft/2.0.0", +] +``` + +Eventually, more nodes will upgrade and therefore start preferring newer protocols, which will change the output. Because we know that all nodes must at least support QBFT v2.0, it becomes the fallback option in the list and the "default" protocol. This way, the Priority protocol will never get stuck and can't produce an empty output. + +The Priority protocol runs once per epoch (the last slot of each epoch) and changes its output depending on the inputs. If another protocol starts to appear at the top of the list, Charon will switch the consensus protocol to that one starting in the next epoch. + +### Changing Consensus Protocol Preference + +A cluster creator can specify the preferred consensus protocol in the cluster configuration file. This new field `consensus_protocol` appeared in the cluster definition file from v1.9 onwards. The field is optional and if not specified, the cluster definition will not impact the consensus protocol selection. + +A node operator can also specify the preferred consensus protocol using the new CLI flag `--consensus-protocol` which has the same effect as the cluster configuration file, but it has a higher precedence. The flag is also optional. + +In both cases, a user is supposed to specify the protocol family name, e.g. `abft` string and not a fully-qualified protocol ID. +The precise version of the protocol is to be determined by the Priority protocol, which will try picking the latest version. +To list all available consensus protocols (with versions), a user can run the command `charon version --verbose`. + +When a node starts, it sequentially mutates the list of preferred consensus protocols by processing the cluster configuration file and then the mentioned CLI flag. The final list of preferred protocols is then passed to the Priority protocol for cluster-wide consensus. Until the Priority protocol reaches consensus, the cluster will use the default QBFT v2.0 protocol for any duties. + +## Observability + +The four existing metrics are reflecting the consensus layer behavior: + +- `core_consensus_decided_rounds` +- `core_consensus_decided_leader_index` +- `core_consensus_duration_seconds` +- `core_consensus_error_total` +- `core_consensus_timeout_total` + +With the new capability to run different consensus protocols, all these metrics now populate the `protocol` label which allows distinguishing between different protocols. +Note that a cluster may run at most two different consensus protocols at the same time, e.g. QBFT v2.0 for Priority and HotStuff v1.0 for validator duties. But this can be changed in the future and more different protocols can be running at the same time. +Therefore the mentioned metrics may have different unique values for the `protocol` label. + +Some protocols may export their own metrics. We agreed that all such metrics should be prefixed with the protocol name, e.g. `core_consensus_hotstuff_xyz`. + +## Debugging + +Charon handles `/debug/consensus` HTTP endpoint that responds with `consensus_messages.pb.gz` file containing certain number of the last consensus messages (in protobuf format). +All consensus messages are tagged with the corresponding protocol ID, in case of multiple protocols running at the same time. + +## Protocol Specific Configuration + +Each consensus protocol may have its own configuration parameters. For instance, QBFT v2.0 has two parameters: `eager_double_linear` and `consensus_participate` that users control via Feature set. +For future protocols we decided to follow the same design and allow users to control the protocol-specific parameters via Feature set. +Charon will set the recommended default values to all such parameters, so node operators don't need to override them unless they know what they are doing. Note that Priority protocol does not take into account any variations caused by different parameters, therefore node operators must be careful when changing them and make sure all nodes have the same configuration. diff --git a/docs/metrics.md b/docs/metrics.md index aa8e0e771..e91751ef6 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -43,11 +43,11 @@ when storing metrics from multiple nodes or clusters in one Prometheus instance. | `core_bcast_recast_errors_total` | Counter | The total count of failed recasted registrations by source; `pregen` vs `downstream` | `source` | | `core_bcast_recast_registration_total` | Counter | The total number of unique validator registration stored in recaster per pubkey | `pubkey` | | `core_bcast_recast_total` | Counter | The total count of recasted registrations by source; `pregen` vs `downstream` | `source` | -| `core_consensus_decided_leader_index` | Gauge | Leader node index of the decision round by duty. | `duty` | -| `core_consensus_decided_rounds` | Gauge | Number of rounds it took to decide consensus instances by duty and timer type. | `duty, timer` | -| `core_consensus_duration_seconds` | Histogram | Duration of a consensus instance in seconds by duty and timer type. | `duty, timer` | -| `core_consensus_error_total` | Counter | Total count of consensus errors | | -| `core_consensus_timeout_total` | Counter | Total count of consensus timeouts by duty and timer type. | `duty, timer` | +| `core_consensus_decided_leader_index` | Gauge | Index of the decided leader by protocol and duty | `protocol, duty` | +| `core_consensus_decided_rounds` | Gauge | Number of decided rounds by protocol, duty, and timer | `protocol, duty, timer` | +| `core_consensus_duration_seconds` | Histogram | Duration of the consensus process by protocol, duty, and timer | `protocol, duty, timer` | +| `core_consensus_error_total` | Counter | Total count of consensus errors by protocol | `protocol` | +| `core_consensus_timeout_total` | Counter | Total count of consensus timeouts by protocol, duty, and timer | `protocol, duty, timer` | | `core_parsigdb_exit_total` | Counter | Total number of partially signed voluntary exits per public key | `pubkey` | | `core_scheduler_current_epoch` | Gauge | The current epoch | | | `core_scheduler_current_slot` | Gauge | The current slot | | diff --git a/p2p/sender_internal_test.go b/p2p/sender_internal_test.go index a2a972f2a..3b8015605 100644 --- a/p2p/sender_internal_test.go +++ b/p2p/sender_internal_test.go @@ -138,7 +138,7 @@ func TestProtocolPrefix(b *testing.T) { func TestIsZeroProto(t *testing.T) { for _, msg := range []proto.Message{ new(pbv1.Duty), - new(pbv1.ConsensusMsg), + new(pbv1.QBFTConsensusMsg), new(timestamppb.Timestamp), } { require.False(t, isZeroProto(nil))