From cdf9e83087573e41a0bcc9db6614b385bc68d890 Mon Sep 17 00:00:00 2001 From: Shota Date: Thu, 24 Oct 2024 01:44:46 +0400 Subject: [PATCH 1/9] get metrics for the number of indices in aggregate and proof (#12389) Co-authored-by: shota.silagadze --- cl/monitor/metrics.go | 10 ++++++++++ .../network/services/aggregate_and_proof_service.go | 2 ++ 2 files changed, 12 insertions(+) diff --git a/cl/monitor/metrics.go b/cl/monitor/metrics.go index dca78f3c33c..a035d285ed8 100644 --- a/cl/monitor/metrics.go +++ b/cl/monitor/metrics.go @@ -19,6 +19,8 @@ var ( metricProposerHit = metrics.GetOrCreateCounter("validator_proposal_hit") // metricProposerMiss is the number of proposals that miss for those validators we observe in previous slot metricProposerMiss = metrics.GetOrCreateCounter("validator_proposal_miss") + // aggregateAndProofSignatures is the sum of signatures in all the aggregates in the recent slot + aggregateAndProofSignatures = metrics.GetOrCreateGauge("aggregate_and_proof_signatures") // Block processing metrics fullBlockProcessingTime = metrics.GetOrCreateGauge("full_block_processing_time") @@ -122,6 +124,11 @@ func microToMilli(micros int64) float64 { return float64(micros) / 1000 } +// ObserveNumberOfAggregateSignatures sets the average processing time for each attestation in aggregate +func ObserveNumberOfAggregateSignatures(signatures int) { + aggregateAndProofSignatures.Add(float64(signatures)) +} + // ObserveEpochProcessingTime sets last epoch processing time func ObserveEpochProcessingTime(startTime time.Time) { epochProcessingTime.Set(float64(time.Since(startTime).Microseconds())) @@ -218,6 +225,9 @@ func ObserveActiveValidatorsCount(count int) { } func ObserveCurrentSlot(slot uint64) { + if currentSlot.GetValueUint64() != slot { + aggregateAndProofSignatures.Set(0) + } currentSlot.Set(float64(slot)) } diff --git a/cl/phase1/network/services/aggregate_and_proof_service.go b/cl/phase1/network/services/aggregate_and_proof_service.go index 4957d116404..6ccd3fe9659 100644 --- a/cl/phase1/network/services/aggregate_and_proof_service.go +++ b/cl/phase1/network/services/aggregate_and_proof_service.go @@ -172,6 +172,8 @@ func (a *aggregateAndProofServiceImpl) ProcessMessage( return errors.New("no attesting indicies") } + monitor.ObserveNumberOfAggregateSignatures(len(attestingIndices)) + // [REJECT] The aggregator's validator index is within the committee -- i.e. aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, index). if !slices.Contains(committee, aggregateAndProof.SignedAggregateAndProof.Message.AggregatorIndex) { return errors.New("committee index not in committee") From 17e9b48bfbc5d0ac9acff470f27f12fc27ae8c9f Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 24 Oct 2024 05:36:44 +0200 Subject: [PATCH 2/9] Remove engine_exchangeTransitionConfiguration (#12430) See https://hackmd.io/@n0ble/deprecate-exchgTC --- .../workflows/qa-rpc-integration-tests.yml | 1 - cmd/rpcdaemon/README.md | 1 - turbo/engineapi/engine_server.go | 25 ------------------- turbo/engineapi/interface.go | 1 - 4 files changed, 28 deletions(-) diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index 6c4a847b698..0caf777dae2 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -87,7 +87,6 @@ jobs: # Run RPC integration test runner via http python3 ./run_tests.py -p 8545 --continue -f --json-diff -x debug_,\ engine_exchangeCapabilities/test_1.json,\ - engine_exchangeTransitionConfigurationV1/test_01.json,\ engine_getClientVersionV1/test_1.json,\ erigon_getLogsByHash/test_04.json,\ erigon_getHeaderByHash/test_02.json,\ diff --git a/cmd/rpcdaemon/README.md b/cmd/rpcdaemon/README.md index 275a7b72fc4..adc4e9df5c1 100644 --- a/cmd/rpcdaemon/README.md +++ b/cmd/rpcdaemon/README.md @@ -324,7 +324,6 @@ The following table shows the current implementation status of Erigon's RPC daem | engine_getPayloadV1 | Yes | | | engine_getPayloadV2 | Yes | | | engine_getPayloadV3 | Yes | | -| engine_exchangeTransitionConfigurationV1 | Yes | | | | | | | debug_accountRange | Yes | Private Erigon debug module | | debug_accountAt | Yes | Private Erigon debug module | diff --git a/turbo/engineapi/engine_server.go b/turbo/engineapi/engine_server.go index 6e51d9a1651..a824fbd9dc0 100644 --- a/turbo/engineapi/engine_server.go +++ b/turbo/engineapi/engine_server.go @@ -737,30 +737,6 @@ func (e *EngineServer) NewPayloadV4(ctx context.Context, payload *engine_types.E return e.newPayload(ctx, payload, expectedBlobHashes, parentBeaconBlockRoot, executionRequests, clparams.ElectraVersion) } -// Receives consensus layer's transition configuration and checks if the execution layer has the correct configuration. -// Can also be used to ping the execution layer (heartbeats). -// See https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.1/src/engine/specification.md#engine_exchangetransitionconfigurationv1 -func (e *EngineServer) ExchangeTransitionConfigurationV1(ctx context.Context, beaconConfig *engine_types.TransitionConfiguration) (*engine_types.TransitionConfiguration, error) { - terminalTotalDifficulty := e.config.TerminalTotalDifficulty - if e.caplin { - e.logger.Crit(caplinEnabledLog) - return nil, errCaplinEnabled - } - if terminalTotalDifficulty == nil { - return nil, fmt.Errorf("the execution layer doesn't have a terminal total difficulty. expected: %v", beaconConfig.TerminalTotalDifficulty) - } - - if terminalTotalDifficulty.Cmp((*big.Int)(beaconConfig.TerminalTotalDifficulty)) != 0 { - return nil, fmt.Errorf("the execution layer has a wrong terminal total difficulty. expected %v, but instead got: %d", beaconConfig.TerminalTotalDifficulty, terminalTotalDifficulty) - } - - return &engine_types.TransitionConfiguration{ - TerminalTotalDifficulty: (*hexutil.Big)(terminalTotalDifficulty), - TerminalBlockHash: libcommon.Hash{}, - TerminalBlockNumber: (*hexutil.Big)(libcommon.Big0), - }, nil -} - // Returns an array of execution payload bodies referenced by their block hashes // See https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1 func (e *EngineServer) GetPayloadBodiesByHashV1(ctx context.Context, hashes []libcommon.Hash) ([]*engine_types.ExecutionPayloadBody, error) { @@ -797,7 +773,6 @@ var ourCapabilities = []string{ "engine_getPayloadV2", "engine_getPayloadV3", "engine_getPayloadV4", - "engine_exchangeTransitionConfigurationV1", "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByHashV2", "engine_getPayloadBodiesByRangeV1", diff --git a/turbo/engineapi/interface.go b/turbo/engineapi/interface.go index 7052b256c87..35d5334e377 100644 --- a/turbo/engineapi/interface.go +++ b/turbo/engineapi/interface.go @@ -38,7 +38,6 @@ type EngineAPI interface { GetPayloadV2(ctx context.Context, payloadID hexutility.Bytes) (*engine_types.GetPayloadResponse, error) GetPayloadV3(ctx context.Context, payloadID hexutility.Bytes) (*engine_types.GetPayloadResponse, error) GetPayloadV4(ctx context.Context, payloadID hexutility.Bytes) (*engine_types.GetPayloadResponse, error) - ExchangeTransitionConfigurationV1(ctx context.Context, transitionConfiguration *engine_types.TransitionConfiguration) (*engine_types.TransitionConfiguration, error) GetPayloadBodiesByHashV1(ctx context.Context, hashes []common.Hash) ([]*engine_types.ExecutionPayloadBody, error) GetPayloadBodiesByHashV2(ctx context.Context, hashes []common.Hash) ([]*engine_types.ExecutionPayloadBody, error) GetPayloadBodiesByRangeV1(ctx context.Context, start, count hexutil.Uint64) ([]*engine_types.ExecutionPayloadBody, error) From 89dbd89f35f91677acc4a8a90bf4ca749d85728d Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 24 Oct 2024 05:37:20 +0200 Subject: [PATCH 3/9] Remove remnants of requests from block body downloader (#12426) Block bodies no longer have requests after PR #12370 --- p2p/sentry/sentry_multi_client/sentry_multi_client.go | 2 +- turbo/stages/bodydownload/body_algos.go | 10 +++------- turbo/stages/bodydownload/body_data_struct.go | 5 ++--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client.go b/p2p/sentry/sentry_multi_client/sentry_multi_client.go index c15a91a2889..b0c6a3b2776 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client.go @@ -475,7 +475,7 @@ func (cs *MultiClient) blockBodies66(ctx context.Context, inreq *proto_sentry.In // No point processing empty response return nil } - cs.Bd.DeliverBodies(txs, uncles, withdrawals, nil, uint64(len(inreq.Data)), sentry.ConvertH512ToPeerID(inreq.PeerId)) + cs.Bd.DeliverBodies(txs, uncles, withdrawals, uint64(len(inreq.Data)), sentry.ConvertH512ToPeerID(inreq.PeerId)) return nil } diff --git a/turbo/stages/bodydownload/body_algos.go b/turbo/stages/bodydownload/body_algos.go index eac4c1388a3..509af2bc67b 100644 --- a/turbo/stages/bodydownload/body_algos.go +++ b/turbo/stages/bodydownload/body_algos.go @@ -164,8 +164,7 @@ func (bd *BodyDownload) RequestMoreBodies(tx kv.RwTx, blockReader services.FullB } if request { if header.UncleHash == types.EmptyUncleHash && header.TxHash == types.EmptyRootHash && - (header.WithdrawalsHash == nil || *header.WithdrawalsHash == types.EmptyRootHash) && - (header.RequestsHash == nil || *header.RequestsHash == types.EmptyRequestsHash) { + (header.WithdrawalsHash == nil || *header.WithdrawalsHash == types.EmptyRootHash) { // Empty block body body := &types.RawBody{} if header.WithdrawalsHash != nil { @@ -192,14 +191,11 @@ func (bd *BodyDownload) RequestMoreBodies(tx kv.RwTx, blockReader services.FullB if header.WithdrawalsHash != nil { copy(bodyHashes[2*length.Hash:], header.WithdrawalsHash.Bytes()) } - if header.RequestsHash != nil { - copy(bodyHashes[3*length.Hash:], header.RequestsHash.Bytes()) - } bd.requestedMap[bodyHashes] = blockNum blockNums = append(blockNums, blockNum) hashes = append(hashes, hash) } else { - // uncleHash, txHash, withdrawalsHash, and requestsHash are all empty (or block is prefetched), no need to request + // uncleHash, txHash, and withdrawalsHash are all empty (or block is prefetched), no need to request bd.delivered.Add(blockNum) } } @@ -253,7 +249,7 @@ func (bd *BodyDownload) RequestSent(bodyReq *BodyRequest, timeWithTimeout uint64 // DeliverBodies takes the block body received from a peer and adds it to the various data structures func (bd *BodyDownload) DeliverBodies(txs [][][]byte, uncles [][]*types.Header, withdrawals []types.Withdrawals, - requests []types.FlatRequests, lenOfP2PMsg uint64, peerID [64]byte, + lenOfP2PMsg uint64, peerID [64]byte, ) { bd.deliveryCh <- Delivery{txs: txs, uncles: uncles, withdrawals: withdrawals, lenOfP2PMessage: lenOfP2PMsg, peerID: peerID} diff --git a/turbo/stages/bodydownload/body_data_struct.go b/turbo/stages/bodydownload/body_data_struct.go index 71cf5d2e991..9d233ed1700 100644 --- a/turbo/stages/bodydownload/body_data_struct.go +++ b/turbo/stages/bodydownload/body_data_struct.go @@ -29,8 +29,8 @@ import ( "github.com/erigontech/erigon/core/types" ) -// BodyHashes is to be used for the mapping between TxHash, UncleHash, WithdrawalsHash, and RequestRoot to the block header -type BodyHashes [4 * length.Hash]byte +// BodyHashes is to be used for the mapping between TxHash, UncleHash, and WithdrawalsHash to the block header +type BodyHashes [3 * length.Hash]byte const MaxBodiesInRequest = 1024 @@ -39,7 +39,6 @@ type Delivery struct { txs [][][]byte uncles [][]*types.Header withdrawals []types.Withdrawals - requests []types.FlatRequests lenOfP2PMessage uint64 } From fe19f3dd3a784d584dd17974e8eeb37d63f97c34 Mon Sep 17 00:00:00 2001 From: Michelangelo Riccobene Date: Thu, 24 Oct 2024 05:38:18 +0200 Subject: [PATCH 4/9] qa-tests: rpc bisection tool - fix version tag (#12344) --- .github/workflows/qa-rpc-test-bisection-tool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qa-rpc-test-bisection-tool.yml b/.github/workflows/qa-rpc-test-bisection-tool.yml index 282c9faa667..46ef650b9c0 100644 --- a/.github/workflows/qa-rpc-test-bisection-tool.yml +++ b/.github/workflows/qa-rpc-test-bisection-tool.yml @@ -39,7 +39,7 @@ jobs: - name: Checkout RPC Tests Repository & Install Requirements run: | rm -rf $GITHUB_WORKSPACE/rpc-tests - git -c advice.detachedHead=false clone --depth 1 --branch v1.00.0 https://github.com/erigontech/rpc-tests $GITHUB_WORKSPACE/rpc-tests + git -c advice.detachedHead=false clone --depth 1 --branch v1.0.0 https://github.com/erigontech/rpc-tests $GITHUB_WORKSPACE/rpc-tests cd $GITHUB_WORKSPACE/rpc-tests pip3 install -r requirements.txt From 009fc192817ebf736aaf571c1fca6ade8b882901 Mon Sep 17 00:00:00 2001 From: steven Date: Thu, 24 Oct 2024 11:42:53 +0800 Subject: [PATCH 5/9] speed up closeWhatNotInList (#12114) 1. accelerate scanning files by avoiding doubly nested loop 2. ~~do not reopen~~ (achieved by #12332 and #12362, so remove relevant code here) --- erigon-lib/kv/mdbx/kv_mdbx.go | 2 +- erigon-lib/state/domain.go | 11 ++++++---- erigon-lib/state/history.go | 13 ++++++----- erigon-lib/state/inverted_index.go | 12 ++++++---- eth/stagedsync/stage.go | 2 +- .../freezeblocks/block_snapshots.go | 13 +++++------ .../freezeblocks/caplin_snapshots.go | 22 +++++++++---------- 7 files changed, 41 insertions(+), 34 deletions(-) diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index e0feb87a227..c7283200362 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -68,7 +68,7 @@ type MdbxOpts struct { shrinkThreshold int flags uint pageSize uint64 - dirtySpace uint64 // if exeed this space, modified pages will `spill` to disk + dirtySpace uint64 // if exceed this space, modified pages will `spill` to disk mergeThreshold uint64 verbosity kv.DBVerbosityLvl label kv.Label // marker to distinct db instances - one process may open many databases. for example to collect metrics of only 1 database diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 23554a3df69..a2dd099e924 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -466,13 +466,16 @@ func (d *Domain) openDirtyFiles() (err error) { } func (d *Domain) closeWhatNotInList(fNames []string) { + protectFiles := make(map[string]struct{}, len(fNames)) + for _, f := range fNames { + protectFiles[f] = struct{}{} + } var toClose []*filesItem d.dirtyFiles.Walk(func(items []*filesItem) bool { - Loop1: for _, item := range items { - for _, protectName := range fNames { - if item.decompressor != nil && item.decompressor.FileName() == protectName { - continue Loop1 + if item.decompressor != nil { + if _, ok := protectFiles[item.decompressor.FileName()]; ok { + continue } } toClose = append(toClose, item) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index c2b0ffd5d6e..fac1beec9e7 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -76,7 +76,7 @@ type History struct { compressCfg seg.Cfg compression seg.FileCompression - //TODO: re-visit this check - maybe we don't need it. It's abot kill in the middle of merge + //TODO: re-visit this check - maybe we don't need it. It's about kill in the middle of merge integrityCheck func(fromStep, toStep uint64) bool // not large: @@ -292,13 +292,16 @@ func (h *History) openDirtyFiles() error { } func (h *History) closeWhatNotInList(fNames []string) { + protectFiles := make(map[string]struct{}, len(fNames)) + for _, f := range fNames { + protectFiles[f] = struct{}{} + } var toClose []*filesItem h.dirtyFiles.Walk(func(items []*filesItem) bool { - Loop1: for _, item := range items { - for _, protectName := range fNames { - if item.decompressor != nil && item.decompressor.FileName() == protectName { - continue Loop1 + if item.decompressor != nil { + if _, ok := protectFiles[item.decompressor.FileName()]; ok { + continue } } toClose = append(toClose, item) diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 9aba57d41e0..fab8c6dd9ea 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -348,15 +348,19 @@ func (ii *InvertedIndex) openDirtyFiles() error { } func (ii *InvertedIndex) closeWhatNotInList(fNames []string) { + protectFiles := make(map[string]struct{}, len(fNames)) + for _, f := range fNames { + protectFiles[f] = struct{}{} + } var toClose []*filesItem ii.dirtyFiles.Walk(func(items []*filesItem) bool { - Loop1: for _, item := range items { - for _, protectName := range fNames { - if item.decompressor != nil && item.decompressor.FileName() == protectName { - continue Loop1 + if item.decompressor != nil { + if _, ok := protectFiles[item.decompressor.FileName()]; ok { + continue } } + toClose = append(toClose, item) } return true diff --git a/eth/stagedsync/stage.go b/eth/stagedsync/stage.go index ce651474132..f9a4b27dbe2 100644 --- a/eth/stagedsync/stage.go +++ b/eth/stagedsync/stage.go @@ -87,7 +87,7 @@ func (s *StageState) ExecutionAt(db kv.Getter) (uint64, error) { } type UnwindReason struct { - // If we;re unwinding due to a fork - we want to unlink blocks but not mark + // If we're unwinding due to a fork - we want to unlink blocks but not mark // them as bad - as they may get replayed then deselected Block *libcommon.Hash // If unwind is caused by a bad block, this error is not empty diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index 95be4e269b2..0593d8453e7 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -933,23 +933,22 @@ func (s *RoSnapshots) Close() { } func (s *RoSnapshots) closeWhatNotInList(l []string) { + protectFiles := make(map[string]struct{}, len(l)) + for _, f := range l { + protectFiles[f] = struct{}{} + } toClose := make(map[snaptype.Enum][]*DirtySegment, 0) s.segments.Scan(func(segtype snaptype.Enum, value *segments) bool { value.DirtySegments.Walk(func(segs []*DirtySegment) bool { - - Loop1: for _, seg := range segs { - for _, fName := range l { - if fName == seg.FileName() { - continue Loop1 - } + if _, ok := protectFiles[seg.FileName()]; ok { + continue } if _, ok := toClose[seg.segType.Enum()]; !ok { toClose[segtype] = make([]*DirtySegment, 0) } toClose[segtype] = append(toClose[segtype], seg) } - return true }) return true diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go index 2329004313f..dfebdf600a8 100644 --- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go @@ -379,18 +379,19 @@ func (s *CaplinSnapshots) OpenFolder() error { } func (s *CaplinSnapshots) closeWhatNotInList(l []string) { + protectFiles := make(map[string]struct{}, len(l)) + for _, fName := range l { + protectFiles[fName] = struct{}{} + } toClose := make([]*DirtySegment, 0) s.BeaconBlocks.DirtySegments.Walk(func(segments []*DirtySegment) bool { - Loop1: for _, sn := range segments { if sn.Decompressor == nil { - continue Loop1 + continue } _, name := filepath.Split(sn.FilePath()) - for _, fName := range l { - if fName == name { - continue Loop1 - } + if _, ok := protectFiles[name]; ok { + continue } toClose = append(toClose, sn) } @@ -403,16 +404,13 @@ func (s *CaplinSnapshots) closeWhatNotInList(l []string) { toClose = make([]*DirtySegment, 0) s.BlobSidecars.DirtySegments.Walk(func(segments []*DirtySegment) bool { - Loop2: for _, sn := range segments { if sn.Decompressor == nil { - continue Loop2 + continue } _, name := filepath.Split(sn.FilePath()) - for _, fName := range l { - if fName == name { - continue Loop2 - } + if _, ok := protectFiles[name]; ok { + continue } toClose = append(toClose, sn) } From dbefdad8c10ffe80acbde30c1f4fc27a76f73ccb Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Thu, 24 Oct 2024 05:50:21 +0200 Subject: [PATCH 6/9] txn overlap (#12433) Fixed ```MDBX_TXN_OVERLAPPING: Overlapping read and write transactions for the current thread, label: chaindata, trace: [kv_mdbx.go:782 kv_mdbx.go:935 aggregator.go:1959 aggregator.go:1635 exec3.go:369 stage_execute.go:164 stage_execute.go:254 default_stages.go:238 sync.go:531 sync.go:410 forkchoice.go:453 asm_amd64.s:1695]"``` --------- Co-authored-by: alex.sharov --- erigon-lib/state/aggregator.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 6d8de00bb38..744ea82b553 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -1631,12 +1631,7 @@ func (a *Aggregator) BuildFilesInBackground(txNum uint64) chan struct{} { } step := a.visibleFilesMinimaxTxNum.Load() / a.StepSize() - lastInDB := max( - lastIdInDB(a.db, a.d[kv.AccountsDomain]), - lastIdInDB(a.db, a.d[kv.CodeDomain]), - lastIdInDB(a.db, a.d[kv.StorageDomain]), - lastIdInDBNoHistory(a.db, a.d[kv.CommitmentDomain])) - log.Info("BuildFilesInBackground", "step", step, "lastInDB", lastInDB) + a.wg.Add(1) go func() { defer a.wg.Done() @@ -1656,6 +1651,7 @@ func (a *Aggregator) BuildFilesInBackground(txNum uint64) chan struct{} { lastIdInDB(a.db, a.d[kv.CodeDomain]), lastIdInDB(a.db, a.d[kv.StorageDomain]), lastIdInDBNoHistory(a.db, a.d[kv.CommitmentDomain])) + log.Info("BuildFilesInBackground", "step", step, "lastInDB", lastInDB) // check if db has enough data (maybe we didn't commit them yet or all keys are unique so history is empty) //lastInDB := lastIdInDB(a.db, a.d[kv.AccountsDomain]) From 5fc055fa1ee2905033ceebc8b9fdd47073f4a58d Mon Sep 17 00:00:00 2001 From: Michelangelo Riccobene Date: Thu, 24 Oct 2024 09:04:56 +0200 Subject: [PATCH 7/9] rpc-tests: remove test on engine_exchangetransitionconfigurationv1 (#12438) Due to this commit: https://github.com/erigontech/erigon/pull/12430 --- .github/workflows/qa-rpc-integration-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index 0caf777dae2..6c4a847b698 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -87,6 +87,7 @@ jobs: # Run RPC integration test runner via http python3 ./run_tests.py -p 8545 --continue -f --json-diff -x debug_,\ engine_exchangeCapabilities/test_1.json,\ + engine_exchangeTransitionConfigurationV1/test_01.json,\ engine_getClientVersionV1/test_1.json,\ erigon_getLogsByHash/test_04.json,\ erigon_getHeaderByHash/test_02.json,\ From 5ca9e25077d812b867e417ad7e621e124c1e2256 Mon Sep 17 00:00:00 2001 From: Somnath Date: Thu, 24 Oct 2024 12:44:04 +0530 Subject: [PATCH 8/9] Simplify EIP-6110 Deposit requests (#12388) Use FlatRequest instead of a different type - DepositRequest defined earlier. This allows all request to align, with the differences in Deposit Requests separated out into functions rather than receiver methods. Tasks board: https://github.com/erigontech/erigon/issues/12106 --- consensus/merge/merge.go | 5 +- consensus/misc/eip6110.go | 85 ++++++++++++++ core/types/deposit_request.go | 195 --------------------------------- core/types/eip7685_requests.go | 2 + core/types/encdec_test.go | 56 ---------- 5 files changed, 89 insertions(+), 254 deletions(-) create mode 100644 consensus/misc/eip6110.go delete mode 100644 core/types/deposit_request.go diff --git a/consensus/merge/merge.go b/consensus/merge/merge.go index fc91e1bf878..bc54bd54942 100644 --- a/consensus/merge/merge.go +++ b/consensus/merge/merge.go @@ -192,11 +192,11 @@ func (s *Merge) Finalize(config *chain.Config, header *types.Header, state *stat for _, rec := range receipts { allLogs = append(allLogs, rec.Logs...) } - depositReqs, err := types.ParseDepositLogs(allLogs, config.DepositContract) + depositReqs, err := misc.ParseDepositLogs(allLogs, config.DepositContract) if err != nil { return nil, nil, nil, fmt.Errorf("error: could not parse requests logs: %v", err) } - rs = append(rs, types.FlatRequest{Type: types.DepositRequestType, RequestData: depositReqs.Encode()}) + rs = append(rs, *depositReqs) withdrawalReq := misc.DequeueWithdrawalRequests7002(syscall) rs = append(rs, *withdrawalReq) consolidations := misc.DequeueConsolidationRequests7251(syscall) @@ -207,7 +207,6 @@ func (s *Merge) Finalize(config *chain.Config, header *types.Header, state *stat return nil, nil, nil, fmt.Errorf("error: invalid requests root hash in header, expected: %v, got :%v", header.RequestsHash, rh) } } - } return txs, receipts, rs, nil diff --git a/consensus/misc/eip6110.go b/consensus/misc/eip6110.go new file mode 100644 index 00000000000..3c4de44180a --- /dev/null +++ b/consensus/misc/eip6110.go @@ -0,0 +1,85 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package misc + +import ( + "fmt" + + libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon/accounts/abi" + "github.com/erigontech/erigon/core/types" +) + +const ( + BLSPubKeyLen = 48 + WithdrawalCredentialsLen = 32 // withdrawalCredentials size + BLSSigLen = 96 // signature size +) + +var ( + // DepositABI is an ABI instance of beacon chain deposit events. + DepositABI = abi.ABI{Events: map[string]abi.Event{"DepositEvent": depositEvent}} + bytesT, _ = abi.NewType("bytes", "", nil) + depositEvent = abi.NewEvent("DepositEvent", "DepositEvent", false, abi.Arguments{ + {Name: "pubkey", Type: bytesT, Indexed: false}, + {Name: "withdrawal_credentials", Type: bytesT, Indexed: false}, + {Name: "amount", Type: bytesT, Indexed: false}, + {Name: "signature", Type: bytesT, Indexed: false}, + {Name: "index", Type: bytesT, Indexed: false}}, + ) +) + +// field type overrides for abi upacking +type depositUnpacking struct { + Pubkey []byte + WithdrawalCredentials []byte + Amount []byte + Signature []byte + Index []byte +} + +// unpackDepositLog unpacks a serialized DepositEvent. +func unpackDepositLog(data []byte) ([]byte, error) { + var du depositUnpacking + if err := DepositABI.UnpackIntoInterface(&du, "DepositEvent", data); err != nil { + return nil, err + } + reqData := make([]byte, 0, types.DepositRequestDataLen) + reqData = append(reqData, du.Pubkey...) + reqData = append(reqData, du.WithdrawalCredentials...) + reqData = append(reqData, du.Amount...) + reqData = append(reqData, du.Signature...) + reqData = append(reqData, du.Index...) + + return reqData, nil +} + +// ParseDepositLogs extracts the EIP-6110 deposit values from logs emitted by +// BeaconDepositContract and returns a FlatRequest object ptr +func ParseDepositLogs(logs []*types.Log, depositContractAddress libcommon.Address) (*types.FlatRequest, error) { + reqData := make([]byte, 0, len(logs)*types.DepositRequestDataLen) + for _, log := range logs { + if log.Address == depositContractAddress { + d, err := unpackDepositLog(log.Data) + if err != nil { + return nil, fmt.Errorf("unable to parse deposit data: %v", err) + } + reqData = append(reqData, d...) + } + } + return &types.FlatRequest{Type: types.DepositRequestType, RequestData: reqData}, nil +} diff --git a/core/types/deposit_request.go b/core/types/deposit_request.go deleted file mode 100644 index ae09ec5e5df..00000000000 --- a/core/types/deposit_request.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package types - -import ( - "encoding/binary" - "encoding/json" - "errors" - "fmt" - - libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/common/hexutil" - "github.com/erigontech/erigon-lib/common/hexutility" - - "github.com/erigontech/erigon/accounts/abi" -) - -const ( - BLSPubKeyLen = 48 - WithdrawalCredentialsLen = 32 // withdrawalCredentials size - BLSSigLen = 96 // signature size -) - -var ( - // DepositABI is an ABI instance of beacon chain deposit events. - DepositABI = abi.ABI{Events: map[string]abi.Event{"DepositEvent": depositEvent}} - bytesT, _ = abi.NewType("bytes", "", nil) - depositEvent = abi.NewEvent("DepositEvent", "DepositEvent", false, abi.Arguments{ - {Name: "pubkey", Type: bytesT, Indexed: false}, - {Name: "withdrawal_credentials", Type: bytesT, Indexed: false}, - {Name: "amount", Type: bytesT, Indexed: false}, - {Name: "signature", Type: bytesT, Indexed: false}, - {Name: "index", Type: bytesT, Indexed: false}}, - ) -) - -type DepositRequest struct { - Pubkey [BLSPubKeyLen]byte // public key of validator - WithdrawalCredentials libcommon.Hash // beneficiary of the validator - Amount uint64 // deposit size in Gwei - Signature [BLSSigLen]byte // signature over deposit msg - Index uint64 // deposit count value -} - -type DepositRequestJson struct { - Pubkey string `json:"pubkey"` - WithdrawalCredentials libcommon.Hash `json:"withdrawalCredentials"` - Amount hexutil.Uint64 `json:"amount"` - Signature string `json:"signature"` - Index hexutil.Uint64 `json:"index"` -} - -func (d *DepositRequest) RequestType() byte { return DepositRequestType } - -func (d *DepositRequest) Encode() []byte { - b := []byte{} - // b = append(b, DepositRequestType) - b = append(b, d.Pubkey[:]...) - b = append(b, d.WithdrawalCredentials.Bytes()...) - b = binary.LittleEndian.AppendUint64(b, d.Amount) - b = append(b, d.Signature[:]...) - b = binary.LittleEndian.AppendUint64(b, d.Index) - return b -} - -func (d *DepositRequest) copy() *DepositRequest { - return &DepositRequest{ - Pubkey: d.Pubkey, - WithdrawalCredentials: d.WithdrawalCredentials, - Amount: d.Amount, - Signature: d.Signature, - Index: d.Index, - } -} - -func (d *DepositRequest) EncodingSize() (encodingSize int) { - return BLSPubKeyLen + WithdrawalCredentialsLen + 8 + BLSSigLen + 8 // 192 -} - -func (d *DepositRequest) MarshalJSON() ([]byte, error) { - tt := DepositRequestJson{ - Pubkey: hexutility.Encode(d.Pubkey[:]), - WithdrawalCredentials: d.WithdrawalCredentials, - Amount: hexutil.Uint64(d.Amount), - Signature: hexutility.Encode(d.Signature[:]), - Index: hexutil.Uint64(d.Index), - } - return json.Marshal(tt) -} - -func (d *DepositRequest) UnmarshalJSON(input []byte) error { - tt := DepositRequestJson{} - err := json.Unmarshal(input, &tt) - if err != nil { - return err - } - pubkey, err := hexutil.Decode(tt.Pubkey) - if err != nil { - return err - } - if len(pubkey) != BLSPubKeyLen { - return errors.New("DepositRequest Pubkey len not equal to BLSPubkeyLen after UnmarshalJSON") - } - sig, err := hexutil.Decode(tt.Signature) - if err != nil { - return err - } - if len(sig) != BLSSigLen { - return errors.New("DepositRequest Signature len not equal to BLSSiglen after UnmarshalJSON") - } - - d.Pubkey = [BLSPubKeyLen]byte(pubkey) - d.Signature = [BLSSigLen]byte(sig) - d.WithdrawalCredentials = tt.WithdrawalCredentials - d.Amount = tt.Amount.Uint64() - d.Index = tt.Index.Uint64() - return nil -} - -// field type overrides for abi upacking -type depositUnpacking struct { - Pubkey []byte - WithdrawalCredentials []byte - Amount []byte - Signature []byte - Index []byte -} - -// unpackIntoDeposit unpacks a serialized DepositEvent. -func unpackIntoDeposit(data []byte) (*DepositRequest, error) { - var du depositUnpacking - if err := DepositABI.UnpackIntoInterface(&du, "DepositEvent", data); err != nil { - return nil, err - } - var d DepositRequest - copy(d.Pubkey[:], du.Pubkey) - copy(d.WithdrawalCredentials[:], du.WithdrawalCredentials) - d.Amount = binary.LittleEndian.Uint64(du.Amount) - copy(d.Signature[:], du.Signature) - d.Index = binary.LittleEndian.Uint64(du.Index) - - return &d, nil -} - -// ParseDepositLogs extracts the EIP-6110 deposit values from logs emitted by -// BeaconDepositContract. -func ParseDepositLogs(logs []*Log, depositContractAddress libcommon.Address) (DepositRequests, error) { - deposits := DepositRequests{} - for _, log := range logs { - if log.Address == depositContractAddress { - d, err := unpackIntoDeposit(log.Data) - if err != nil { - return nil, fmt.Errorf("unable to parse deposit data: %v", err) - } - deposits = append(deposits, d) - } - } - return deposits, nil -} - -type DepositRequests []*DepositRequest - -// Len returns the length of s. -func (s DepositRequests) Len() int { return len(s) } - -func (s DepositRequests) Encode() []byte { - flatDeposits := make([]byte, 0, len(s)*DepositRequestDataLen) - for _, d := range s { - flatDeposits = append(flatDeposits, d.Encode()...) - } - return flatDeposits -} - -// // Requests creates a deep copy of each deposit and returns a slice of the -// // withdrwawal requests as Request objects. -// func (s DepositRequests) Requests() (reqs Requests) { -// for _, d := range s { -// reqs = append(reqs, d) -// } -// return -// } diff --git a/core/types/eip7685_requests.go b/core/types/eip7685_requests.go index 64af12e1fa6..bb5c31a10ac 100644 --- a/core/types/eip7685_requests.go +++ b/core/types/eip7685_requests.go @@ -31,6 +31,8 @@ const ConsolidationRequestDataLen = 116 // addr + sourcePubkey + targetPubkey var KnownRequestTypes = []byte{DepositRequestType, WithdrawalRequestType, ConsolidationRequestType} +// FlatRequest carries serialized (flat) request data from any known Request type +// The RequestData slice can contain collated data for more than one request of the same type type FlatRequest struct { Type byte RequestData []byte diff --git a/core/types/encdec_test.go b/core/types/encdec_test.go index 0db74b0f0ee..69a19c5a0c1 100644 --- a/core/types/encdec_test.go +++ b/core/types/encdec_test.go @@ -86,25 +86,6 @@ func (tr *TRand) RandWithdrawal() *Withdrawal { } } -func (tr *TRand) RandDepositRequest() *DepositRequest { - return &DepositRequest{ - Pubkey: [48]byte(tr.RandBytes(48)), - WithdrawalCredentials: tr.RandHash(), - Amount: *tr.RandUint64(), - Signature: [96]byte(tr.RandBytes(96)), - Index: *tr.RandUint64(), - } -} - -func (tr *TRand) RandRequest() *DepositRequest { - switch tr.rnd.Intn(3) { - case 0: - return tr.RandDepositRequest() - default: - return nil // unreachable code - } -} - func (tr *TRand) RandHeader() *Header { wHash := tr.RandHash() pHash := tr.RandHash() @@ -273,14 +254,6 @@ func (tr *TRand) RandWithdrawals(size int) []*Withdrawal { return withdrawals } -func (tr *TRand) RandRequests(size int) []FlatRequest { - requests := make([]FlatRequest, size) - for i := 0; i < size; i++ { - requests[i] = FlatRequest{RequestData: tr.RandRequest().Encode(), Type: DepositRequestType} - } - return requests -} - func (tr *TRand) RandRawBody() *RawBody { return &RawBody{ Transactions: tr.RandRawTransactions(tr.RandIntInRange(1, 6)), @@ -382,23 +355,6 @@ func compareTransactions(t *testing.T, a, b Transaction) { check(t, "Tx.S", s1, s2) } -func compareDeposits(t *testing.T, a, b *DepositRequest) { - check(t, "Deposit.Pubkey", a.Pubkey, b.Pubkey) - check(t, "Deposit.WithdrawalCredentials", a.WithdrawalCredentials, b.WithdrawalCredentials) - check(t, "Deposit.Amount", a.Amount, b.Amount) - check(t, "Deposit.Signature", a.Signature, b.Signature) - check(t, "Deposit.Index", a.Index, b.Index) -} - -func checkRequests(t *testing.T, a, b DepositRequest) { - if a.RequestType() != b.RequestType() { - t.Errorf("request type mismatch: request-a: %v, request-b: %v", a.RequestType(), b.RequestType()) - } - - compareDeposits(t, &a, &b) - -} - func compareHeaders(t *testing.T, a, b []*Header) error { auLen, buLen := len(a), len(b) if auLen != buLen { @@ -423,18 +379,6 @@ func compareWithdrawals(t *testing.T, a, b []*Withdrawal) error { return nil } -func compareRequests(t *testing.T, a, b DepositRequests) error { - // arLen, brLen := len(a), len(b) - // if arLen != brLen { - // return fmt.Errorf("requests len mismatch: expected: %v, got: %v", arLen, brLen) - // } - - // for i := 0; i < arLen; i++ { - // checkRequests(t, &a[i], &b[i]) - // } - return nil -} - func compareRawBodies(t *testing.T, a, b *RawBody) error { atLen, btLen := len(a.Transactions), len(b.Transactions) From f9527b88bc459ee10f90d892945167ebd2a9d1f2 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 24 Oct 2024 17:16:20 +0800 Subject: [PATCH 9/9] rpcdaemon: Get state at end of the block when generating Bor receipts (#12436) We want to get the state at the end of the block when executing state sync events. `CreateHistoryStateReader` gives us the state before the transaction at `txIndex` is executed, so we want to get the state at `txIndex=len(transactions)`. --- turbo/jsonrpc/receipts/bor_receipts_generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/jsonrpc/receipts/bor_receipts_generator.go b/turbo/jsonrpc/receipts/bor_receipts_generator.go index 0c717c65e02..6838e6c54e4 100644 --- a/turbo/jsonrpc/receipts/bor_receipts_generator.go +++ b/turbo/jsonrpc/receipts/bor_receipts_generator.go @@ -49,7 +49,7 @@ func (g *BorGenerator) GenerateBorReceipt(ctx context.Context, tx kv.Tx, block * } txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, g.blockReader)) - ibs, blockContext, _, _, _, err := transactions.ComputeBlockContext(ctx, g.engine, block.HeaderNoCopy(), chainConfig, g.blockReader, txNumsReader, tx, len(blockReceipts)-1) + ibs, blockContext, _, _, _, err := transactions.ComputeBlockContext(ctx, g.engine, block.HeaderNoCopy(), chainConfig, g.blockReader, txNumsReader, tx, len(blockReceipts)) // we want to get the state at the end of the block if err != nil { return nil, err }