From 9d270795ed86ed7b57275cb8f290d60b65e72ff4 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Wed, 6 Nov 2024 07:52:28 +0100 Subject: [PATCH 1/9] Minimally lock the state during aggregate processing + memory tuning (#12571) - Changed the positions of lock release in aggregate and proof - Removed diffset because it was taking way too much memory and caused way too many allocs - also it is a data structure only used for some debug endpoints so not worth it - Removed compressor for caches (most of those are random 32-bytes hashes anyway) --- .../fork_graph/diff_storage/diff_storage.go | 131 ------------------ .../diff_storage/diff_storage_test.go | 100 ------------- .../forkchoice/fork_graph/fork_graph_disk.go | 92 +++--------- .../fork_graph/fork_graph_disk_fs.go | 20 +-- 4 files changed, 27 insertions(+), 316 deletions(-) delete mode 100644 cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage.go delete mode 100644 cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage_test.go diff --git a/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage.go b/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage.go deleted file mode 100644 index 330c758e014..00000000000 --- a/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package diffstorage - -import ( - "bytes" - "io" - "sync" - - "github.com/alecthomas/atomic" - libcommon "github.com/erigontech/erigon-lib/common" -) - -const maxDumps = 8 // max number of dumps to keep in memory to prevent from memory leak during long non-finality. - -var bufferPool = sync.Pool{ - New: func() interface{} { - return new(bytes.Buffer) - }, -} - -type link struct { - from libcommon.Hash - to libcommon.Hash -} - -// Memory storage for binary diffs -type ChainDiffStorage struct { - dumps sync.Map - parent sync.Map // maps child -> parent - links sync.Map // maps root -> []links - diffFn func(w io.Writer, old, new []byte) error - applyFn func(in, out []byte, diff []byte, reverse bool) ([]byte, error) - diffs sync.Map - dumpsCount atomic.Int32 // prevent from memory leak during long non-finality. -} - -func NewChainDiffStorage(diffFn func(w io.Writer, old, new []byte) error, applyFn func(in, out []byte, diff []byte, reverse bool) ([]byte, error)) *ChainDiffStorage { - return &ChainDiffStorage{ - diffFn: diffFn, - applyFn: applyFn, - dumpsCount: atomic.NewInt32(0), - } -} - -func (c *ChainDiffStorage) Insert(root, parent libcommon.Hash, prevDump, dump []byte, isDump bool) error { - c.parent.Store(root, parent) - if isDump { - c.dumpsCount.Add(1) - if c.dumpsCount.Load() > maxDumps { - *c = *NewChainDiffStorage(c.diffFn, c.applyFn) - c.dumpsCount.Store(0) - return nil - } - c.dumps.Store(root, libcommon.Copy(dump)) - return nil - } - - buf := bufferPool.Get().(*bytes.Buffer) - defer bufferPool.Put(buf) - buf.Reset() - - if err := c.diffFn(buf, prevDump, dump); err != nil { - return err - } - c.diffs.Store(link{from: parent, to: root}, libcommon.Copy(buf.Bytes())) - - links, _ := c.links.LoadOrStore(parent, []link{}) - c.links.Store(parent, append(links.([]link), link{from: parent, to: root})) - - return nil -} - -func (c *ChainDiffStorage) Get(root libcommon.Hash) ([]byte, error) { - dump, foundDump := c.dumps.Load(root) - if foundDump { - return dump.([]byte), nil - } - currentRoot := root - diffs := [][]byte{} - for !foundDump { - parent, found := c.parent.Load(currentRoot) - if !found { - return nil, nil - } - diff, foundDiff := c.diffs.Load(link{from: parent.(libcommon.Hash), to: currentRoot}) - if !foundDiff { - return nil, nil - } - diffs = append(diffs, diff.([]byte)) - currentRoot = parent.(libcommon.Hash) - dump, foundDump = c.dumps.Load(currentRoot) - } - out := libcommon.Copy(dump.([]byte)) - for i := len(diffs) - 1; i >= 0; i-- { - var err error - out, err = c.applyFn(out, out, diffs[i], false) - if err != nil { - return nil, err - } - } - return out, nil -} - -func (c *ChainDiffStorage) Delete(root libcommon.Hash) { - if _, loaded := c.dumps.LoadAndDelete(root); loaded { - c.dumpsCount.Add(-1) - } - c.parent.Delete(root) - links, ok := c.links.Load(root) - if ok { - for _, link := range links.([]link) { - c.diffs.Delete(link) - } - } - c.links.Delete(root) -} diff --git a/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage_test.go b/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage_test.go deleted file mode 100644 index e4a6835dcd0..00000000000 --- a/cl/phase1/forkchoice/fork_graph/diff_storage/diff_storage_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package diffstorage - -import ( - "math" - "testing" - - libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/cl/cltypes/solid" - "github.com/erigontech/erigon/cl/persistence/base_encoding" - "github.com/stretchr/testify/require" -) - -// 1 -> 2 -> 3 -> 4 -> 5 -// -// | -// --> 6 -func TestDiffStorage(t *testing.T) { - // decleare 5 nodes - node1 := libcommon.Hash{1} - node2 := libcommon.Hash{2} - node3 := libcommon.Hash{3} - node4 := libcommon.Hash{4} - node5 := libcommon.Hash{5} - node6 := libcommon.Hash{6} - - node1Content := []uint64{1, 2, 3, 4, 5} - node2Content := []uint64{1, 2, 3, 4, 5, 6} - node3Content := []uint64{1, 2, 3, 4, 5, 2, 7} - node4Content := []uint64{1, 2, 3, 4, 5, 2, 7, 8} - node5Content := []uint64{1, 6, 8, 4, 5, 2, 7, 8, 9} - node6Content := []uint64{1, 2, 3, 4, 5, 2, 7, 10} - - exp1 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node1Content) - exp2 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node2Content) - exp3 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node3Content) - exp4 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node4Content) - exp5 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node5Content) - exp6 := solid.NewUint64ListSSZFromSlice(math.MaxInt, node6Content) - - enc1, err := exp1.EncodeSSZ(nil) - require.NoError(t, err) - enc2, err := exp2.EncodeSSZ(nil) - require.NoError(t, err) - enc3, err := exp3.EncodeSSZ(nil) - require.NoError(t, err) - enc4, err := exp4.EncodeSSZ(nil) - require.NoError(t, err) - enc5, err := exp5.EncodeSSZ(nil) - require.NoError(t, err) - enc6, err := exp6.EncodeSSZ(nil) - require.NoError(t, err) - - diffStorage := NewChainDiffStorage(base_encoding.ComputeCompressedSerializedUint64ListDiff, base_encoding.ApplyCompressedSerializedUint64ListDiff) - diffStorage.Insert(node1, libcommon.Hash{}, nil, enc1, true) - diffStorage.Insert(node2, node1, enc1, enc2, false) - diffStorage.Insert(node3, node2, enc2, enc3, false) - diffStorage.Insert(node4, node3, enc3, enc4, false) - diffStorage.Insert(node5, node4, enc4, enc5, false) - diffStorage.Insert(node6, node2, enc2, enc6, false) - - d1, err := diffStorage.Get(node1) - require.NoError(t, err) - require.Equal(t, enc1, d1) - - d2, err := diffStorage.Get(node2) - require.NoError(t, err) - require.Equal(t, enc2, d2) - - d3, err := diffStorage.Get(node3) - require.NoError(t, err) - require.Equal(t, enc3, d3) - - d4, err := diffStorage.Get(node4) - require.NoError(t, err) - require.Equal(t, enc4, d4) - - d5, err := diffStorage.Get(node5) - require.NoError(t, err) - require.Equal(t, enc5, d5) - - d6, err := diffStorage.Get(node6) - require.NoError(t, err) - require.Equal(t, enc6, d6) -} diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go index bcef69833ed..3068031bebb 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk.go @@ -23,7 +23,6 @@ import ( "sync" "sync/atomic" - "github.com/klauspost/compress/zstd" "github.com/spf13/afero" libcommon "github.com/erigontech/erigon-lib/common" @@ -34,9 +33,7 @@ import ( "github.com/erigontech/erigon/cl/cltypes" "github.com/erigontech/erigon/cl/cltypes/lightclient_utils" "github.com/erigontech/erigon/cl/cltypes/solid" - "github.com/erigontech/erigon/cl/persistence/base_encoding" "github.com/erigontech/erigon/cl/phase1/core/state" - diffstorage "github.com/erigontech/erigon/cl/phase1/forkchoice/fork_graph/diff_storage" "github.com/erigontech/erigon/cl/transition" "github.com/erigontech/erigon/cl/transition/impl/eth2" ) @@ -48,26 +45,6 @@ type syncCommittees struct { nextSyncCommittee *solid.SyncCommittee } -var compressorPool = sync.Pool{ - New: func() interface{} { - w, err := zstd.NewWriter(nil) - if err != nil { - panic(err) - } - return w - }, -} - -var decompressPool = sync.Pool{ - New: func() interface{} { - r, err := zstd.NewReader(nil) - if err != nil { - panic(err) - } - return r - }, -} - var ErrStateNotFound = errors.New("state not found") type ChainSegmentInsertionResult uint @@ -132,12 +109,9 @@ type forkGraphDisk struct { // for each block root we keep track of the sync committees for head retrieval. syncCommittees sync.Map lightclientBootstraps sync.Map - // diffs storage - balancesStorage *diffstorage.ChainDiffStorage - validatorSetStorage *diffstorage.ChainDiffStorage - inactivityScoresStorage *diffstorage.ChainDiffStorage - previousIndicies sync.Map - currentIndicies sync.Map + + previousIndicies sync.Map + currentIndicies sync.Map // configurations beaconCfg *clparams.BeaconChainConfig @@ -172,23 +146,16 @@ func NewForkGraphDisk(anchorState *state.CachingBeaconState, aferoFs afero.Fs, r farthestExtendingPath[anchorRoot] = true - balancesStorage := diffstorage.NewChainDiffStorage(base_encoding.ComputeCompressedSerializedUint64ListDiff, base_encoding.ApplyCompressedSerializedUint64ListDiff) - validatorSetStorage := diffstorage.NewChainDiffStorage(base_encoding.ComputeCompressedSerializedValidatorSetListDiff, base_encoding.ApplyCompressedSerializedValidatorListDiff) - inactivityScoresStorage := diffstorage.NewChainDiffStorage(base_encoding.ComputeCompressedSerializedUint64ListDiff, base_encoding.ApplyCompressedSerializedUint64ListDiff) - f := &forkGraphDisk{ fs: aferoFs, // current state data currentState: anchorState, // configuration - beaconCfg: anchorState.BeaconConfig(), - genesisTime: anchorState.GenesisTime(), - anchorSlot: anchorState.Slot(), - balancesStorage: balancesStorage, - validatorSetStorage: validatorSetStorage, - inactivityScoresStorage: inactivityScoresStorage, - rcfg: rcfg, - emitter: emitter, + beaconCfg: anchorState.BeaconConfig(), + genesisTime: anchorState.GenesisTime(), + anchorSlot: anchorState.Slot(), + rcfg: rcfg, + emitter: emitter, } f.lowestAvailableBlock.Store(anchorState.Slot()) f.headers.Store(libcommon.Hash(anchorRoot), &anchorHeader) @@ -280,13 +247,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, } blockRewardsCollector := ð2.BlockRewardsCollector{} - var prevDumpBalances, prevValidatorSetDump, prevInactivityScores []byte - epochCross := newState.Slot()/f.beaconCfg.SlotsPerEpoch != block.Slot/f.beaconCfg.SlotsPerEpoch - if (f.rcfg.Beacon || f.rcfg.Validator || f.rcfg.Lighthouse) && !epochCross { - prevDumpBalances = libcommon.Copy(newState.RawBalances()) - prevValidatorSetDump = libcommon.Copy(newState.RawValidatorSet()) - prevInactivityScores = libcommon.Copy(newState.RawInactivityScores()) - } + // Execute the state if invalidBlockErr := transition.TransitionState(newState, signedBlock, blockRewardsCollector, fullValidation); invalidBlockErr != nil { // Add block to list of invalid blocks @@ -302,11 +263,9 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, if block.Version() != clparams.Phase0Version { f.currentIndicies.Store(libcommon.Hash(blockRoot), libcommon.Copy(newState.RawCurrentEpochParticipation())) f.previousIndicies.Store(libcommon.Hash(blockRoot), libcommon.Copy(newState.RawPreviousEpochParticipation())) - f.inactivityScoresStorage.Insert(libcommon.Hash(blockRoot), block.ParentRoot, prevInactivityScores, newState.RawInactivityScores(), epochCross) } f.blockRewards.Store(libcommon.Hash(blockRoot), blockRewardsCollector) - f.balancesStorage.Insert(libcommon.Hash(blockRoot), block.ParentRoot, prevDumpBalances, newState.RawBalances(), epochCross) - f.validatorSetStorage.Insert(libcommon.Hash(blockRoot), block.ParentRoot, prevValidatorSetDump, newState.RawValidatorSet(), epochCross) + period := f.beaconCfg.SyncCommitteePeriod(newState.Slot()) f.syncCommittees.Store(period, syncCommittees{ currentSyncCommittee: newState.CurrentSyncCommittee().Copy(), @@ -474,9 +433,7 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) { f.blockRewards.Delete(root) f.fs.Remove(getBeaconStateFilename(root)) f.fs.Remove(getBeaconStateCacheFilename(root)) - f.balancesStorage.Delete(root) - f.validatorSetStorage.Delete(root) - f.inactivityScoresStorage.Delete(root) + f.previousIndicies.Delete(root) f.currentIndicies.Delete(root) } @@ -529,27 +486,25 @@ func (f *forkGraphDisk) GetLightClientUpdate(period uint64) (*cltypes.LightClien } func (f *forkGraphDisk) GetBalances(blockRoot libcommon.Hash) (solid.Uint64ListSSZ, error) { - b, err := f.balancesStorage.Get(blockRoot) + st, err := f.GetState(blockRoot, true) if err != nil { return nil, err } - if len(b) == 0 { - return nil, nil + if st == nil { + return nil, ErrStateNotFound } - out := solid.NewUint64ListSSZ(int(f.beaconCfg.ValidatorRegistryLimit)) - return out, out.DecodeSSZ(b, 0) + return st.Balances(), nil } func (f *forkGraphDisk) GetInactivitiesScores(blockRoot libcommon.Hash) (solid.Uint64ListSSZ, error) { - b, err := f.inactivityScoresStorage.Get(blockRoot) + st, err := f.GetState(blockRoot, true) if err != nil { return nil, err } - if len(b) == 0 { - return nil, nil + if st == nil { + return nil, ErrStateNotFound } - out := solid.NewUint64ListSSZ(int(f.beaconCfg.ValidatorRegistryLimit)) - return out, out.DecodeSSZ(b, 0) + return st.InactivityScores(), nil } func (f *forkGraphDisk) GetPreviousParticipationIndicies(blockRoot libcommon.Hash) (*solid.ParticipationBitList, error) { @@ -577,13 +532,12 @@ func (f *forkGraphDisk) GetCurrentParticipationIndicies(blockRoot libcommon.Hash } func (f *forkGraphDisk) GetValidatorSet(blockRoot libcommon.Hash) (*solid.ValidatorSet, error) { - b, err := f.validatorSetStorage.Get(blockRoot) + st, err := f.GetState(blockRoot, true) if err != nil { return nil, err } - if len(b) == 0 { - return nil, nil + if st == nil { + return nil, ErrStateNotFound } - out := solid.NewValidatorSet(int(f.beaconCfg.ValidatorRegistryLimit)) - return out, out.DecodeSSZ(b, 0) + return st.ValidatorSet(), nil } diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go index 902426d7801..11a8bc001d1 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go @@ -24,7 +24,6 @@ import ( "os" "github.com/golang/snappy" - "github.com/klauspost/compress/zstd" "github.com/spf13/afero" libcommon "github.com/erigontech/erigon-lib/common" @@ -94,12 +93,7 @@ func (f *forkGraphDisk) readBeaconStateFromDisk(blockRoot libcommon.Hash) (bs *s } defer cacheFile.Close() - reader := decompressPool.Get().(*zstd.Decoder) - defer decompressPool.Put(reader) - - reader.Reset(cacheFile) - - if err := bs.DecodeCaches(reader); err != nil { + if err := bs.DecodeCaches(cacheFile); err != nil { return nil, err } @@ -162,19 +156,13 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot libcommon.Hash, bs *stat } defer cacheFile.Close() - writer := compressorPool.Get().(*zstd.Encoder) - defer compressorPool.Put(writer) - - writer.Reset(cacheFile) - defer writer.Close() - - if err := bs.EncodeCaches(writer); err != nil { + if err := bs.EncodeCaches(cacheFile); err != nil { return err } - if err = writer.Close(); err != nil { + + if err = cacheFile.Sync(); err != nil { return } - err = cacheFile.Sync() return } From 5f65bb8418e15f787b31fdb34cd5e8500c741912 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Wed, 6 Nov 2024 07:03:52 +0000 Subject: [PATCH 2/9] Strongerrunner (#12602) --- .github/workflows/test-integration.yml | 3 ++- tests/block_test.go | 1 + tests/difficulty_test.go | 2 -- tests/exec_spec_test.go | 1 + tests/init_test.go | 2 -- tests/rlp_test.go | 1 - tests/state_test.go | 2 +- tests/transaction_test.go | 3 +-- 8 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index c7462a8d262..f47c4b0dcaa 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -24,6 +24,7 @@ jobs: os: - ubuntu-22.04 - macos-14 + - ubuntu-latest-erigontests-large runs-on: ${{ matrix.os }} steps: @@ -37,7 +38,7 @@ jobs: run: sudo apt update && sudo apt install build-essential - name: test-integration - run: make test-integration + run: GOGC=50 make test-integration tests-windows: strategy: diff --git a/tests/block_test.go b/tests/block_test.go index 269dde4ab78..3003fd707a0 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -53,6 +53,7 @@ func TestBlockchain(t *testing.T) { checkStateRoot := true bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { + t.Parallel() // import pre accounts & construct test genesis block & state root if err := bt.checkFailure(t, test.Run(t, checkStateRoot)); err != nil { t.Error(err) diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index daab9e53e8e..c301d94ff70 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -28,8 +28,6 @@ import ( ) func TestDifficulty(t *testing.T) { - //t.Parallel() - dt := new(testMatcher) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, superTest map[string]json.RawMessage) { diff --git a/tests/exec_spec_test.go b/tests/exec_spec_test.go index d8ea375f79a..e93e660f97b 100644 --- a/tests/exec_spec_test.go +++ b/tests/exec_spec_test.go @@ -35,6 +35,7 @@ func TestExecutionSpec(t *testing.T) { checkStateRoot := true bt.walk(t, dir, func(t *testing.T, name string, test *BlockTest) { + t.Parallel() // import pre accounts & construct test genesis block & state root if err := bt.checkFailure(t, test.Run(t, checkStateRoot)); err != nil { t.Error(err) diff --git a/tests/init_test.go b/tests/init_test.go index a3a28f110f1..27bbcda2ea1 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -228,7 +228,6 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte t.Skip("Skipped by whitelist") } } - //t.Parallel() // Load the file as map[string]. m := makeMapFromTestFunc(runTest) @@ -289,7 +288,6 @@ func runTestFunc(runTest interface{}, t *testing.T, name string, m reflect.Value } func TestMatcherWhitelist(t *testing.T) { - //t.Parallel() tm := new(testMatcher) tm.whitelist("invalid*") tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) { diff --git a/tests/rlp_test.go b/tests/rlp_test.go index f6a907b2ade..25abe33f7e6 100644 --- a/tests/rlp_test.go +++ b/tests/rlp_test.go @@ -26,7 +26,6 @@ import ( ) func TestRLP(t *testing.T) { - //t.Parallel() tm := new(testMatcher) tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) { if err := tm.checkFailure(t, test.Run()); err != nil { diff --git a/tests/state_test.go b/tests/state_test.go index 7a5f9b93ddb..7199c444aac 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -39,12 +39,12 @@ import ( ) func TestState(t *testing.T) { + t.Parallel() defer log.Root().SetHandler(log.Root().GetHandler()) log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StderrHandler)) if runtime.GOOS == "windows" { t.Skip("fix me on win please") // it's too slow on win and stops on macos, need generally improve speed of this tests } - //t.Parallel() st := new(testMatcher) diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 1b3ffd32837..af2b25d0a7b 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -28,8 +28,6 @@ import ( ) func TestTransaction(t *testing.T) { - //t.Parallel() - txt := new(testMatcher) // We don't allow more than uint64 in gas amount @@ -38,6 +36,7 @@ func TestTransaction(t *testing.T) { txt.skipLoad("^ttGasLimit/TransactionWithGasLimitxPriceOverflow.json") txt.walk(t, transactionTestDir, func(t *testing.T, name string, test *TransactionTest) { + t.Parallel() cfg := params.MainnetChainConfig if err := txt.checkFailure(t, test.Run(cfg.ChainID)); err != nil { t.Error(err) From f0af9016d836300f91c0e376b5b49fdc78fd6494 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 6 Nov 2024 14:04:04 +0700 Subject: [PATCH 3/9] jwt dep up (#12635) dependabot recommended --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ff79c4af55e..e3aef287de0 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/go-test/deep v1.1.1 github.com/goccy/go-json v0.9.11 github.com/gofrs/flock v0.12.1 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/google/btree v1.1.3 github.com/google/cel-go v0.18.2 diff --git a/go.sum b/go.sum index 6661c7a2b0c..021e462064e 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From 3d01204109c48481158f3065befe4f895458f32d Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 6 Nov 2024 14:16:22 +0700 Subject: [PATCH 4/9] [rpc-test] enable `debug_traceTransaction` (#12638) --- .github/workflows/qa-rpc-integration-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index 415d0cdf961..ecbcb404a35 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -104,7 +104,6 @@ jobs: # to investigate debug_traceBlockByHash,\ debug_traceCallMany/test_02.tar,debug_traceCallMany/test_04.tar,debug_traceCallMany/test_05.tar,debug_traceCallMany/test_06.tar,debug_traceCallMany/test_07.tar,debug_traceCallMany/test_09.json,debug_traceCallMany/test_10.tar,\ - debug_traceTransaction,\ engine_exchangeCapabilities/test_1.json,\ engine_exchangeTransitionConfigurationV1/test_01.json,\ engine_getClientVersionV1/test_1.json,\ From f948fb4a3b2487924e480f54c1e10d320d0c7ccb Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 6 Nov 2024 15:16:33 +0700 Subject: [PATCH 5/9] `eth_getLogs` to fix `fee cap less than block base fee` error (#12640) --- .github/workflows/lint.yml | 2 +- .github/workflows/qa-rpc-integration-tests.yml | 5 ++--- cmd/state/exec3/trace_worker.go | 5 +++-- core/vm/evm.go | 5 +++++ go.mod | 2 +- turbo/jsonrpc/eth_receipts.go | 4 ---- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3efd756d7e5..31cb5b4a28d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-go@v5 with: - go-version: '1.22' + go-version: '1.23' - name: Install golangci-lint if: runner.os == 'Linux' diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index ecbcb404a35..bc6def09f5a 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -87,6 +87,8 @@ jobs: # Run RPC integration test runner via http python3 ./run_tests.py -p 8545 --continue -f --json-diff -x \ +# Erigon2 and Erigon3 never supported this api methods + trace_rawTransaction,\ # false positives: Erigon return expected response. but rpc-test somehow doesn't see 1 field. erigon_getHeaderByHash,erigon_getHeaderByNumber,eth_feeHistory,\ # total difficulty field was removed, then added back @@ -109,9 +111,6 @@ jobs: engine_getClientVersionV1/test_1.json,\ erigon_getBalanceChangesInBlock,\ eth_createAccessList/test_16.json,\ - trace_filter/test_16.json,\ - trace_rawTransaction/test_01.json,\ - trace_rawTransaction/test_03.json,\ admin_nodeInfo/test_01.json,\ admin_peers/test_01.json,\ erigon_nodeInfo/test_1.json,\ diff --git a/cmd/state/exec3/trace_worker.go b/cmd/state/exec3/trace_worker.go index 7c9ceeb8e79..7b80c49992b 100644 --- a/cmd/state/exec3/trace_worker.go +++ b/cmd/state/exec3/trace_worker.go @@ -74,11 +74,12 @@ func NewTraceWorker(tx kv.TemporalTx, cc *chain.Config, engine consensus.EngineR stateReader: stateReader, tracer: tracer, evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, cc, vm.Config{}), - vmConfig: &vm.Config{}, + vmConfig: &vm.Config{NoBaseFee: true}, ibs: state.New(stateReader), } if tracer != nil { - ie.vmConfig = &vm.Config{Debug: true, Tracer: tracer} + ie.vmConfig.Debug = true + ie.vmConfig.Tracer = tracer } return ie } diff --git a/core/vm/evm.go b/core/vm/evm.go index 1ec0c0ff645..fb46915fed3 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -134,6 +134,11 @@ func (evm *EVM) Reset(txCtx evmtypes.TxContext, ibs evmtypes.IntraBlockState) { } func (evm *EVM) ResetBetweenBlocks(blockCtx evmtypes.BlockContext, txCtx evmtypes.TxContext, ibs evmtypes.IntraBlockState, vmConfig Config, chainRules *chain.Rules) { + if vmConfig.NoBaseFee { + if txCtx.GasPrice.IsZero() { + blockCtx.BaseFee = new(uint256.Int) + } + } evm.Context = blockCtx evm.TxContext = txCtx evm.intraBlockState = ibs diff --git a/go.mod b/go.mod index e3aef287de0..be8789ea804 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/Giulio2002/bls v0.0.0-20241013174947-019133587795 github.com/Masterminds/sprig/v3 v3.2.3 github.com/RoaringBitmap/roaring v1.9.4 - github.com/alecthomas/atomic v0.1.0-alpha2 github.com/alecthomas/kong v0.8.1 github.com/anacrolix/sync v0.5.1 github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8 @@ -107,6 +106,7 @@ require ( ) require ( + github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/elastic/go-freelru v0.13.0 // indirect github.com/erigontech/speedtest v0.0.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/turbo/jsonrpc/eth_receipts.go b/turbo/jsonrpc/eth_receipts.go index 3dbb7b5b69a..ba23694e56a 100644 --- a/turbo/jsonrpc/eth_receipts.go +++ b/turbo/jsonrpc/eth_receipts.go @@ -303,10 +303,6 @@ func (api *BaseAPI) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end continue } blockHash = header.Hash() - - if err != nil { - return nil, err - } exec.ChangeBlock(header) } From 821f3c1c5694d6bffec8389748f3196e7b0e9abf Mon Sep 17 00:00:00 2001 From: lystopad Date: Wed, 6 Nov 2024 09:38:52 +0100 Subject: [PATCH 6/9] Cleanup old snapshot docker images (main-xxxxxxx). (#12610) Keep only latest 100 docker images with tag matching pattern main-XXXXXXX --- .../ci-cd-main-branch-docker-images.yml | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd-main-branch-docker-images.yml b/.github/workflows/ci-cd-main-branch-docker-images.yml index bdfc1e6a883..b476d0db4c9 100644 --- a/.github/workflows/ci-cd-main-branch-docker-images.yml +++ b/.github/workflows/ci-cd-main-branch-docker-images.yml @@ -9,6 +9,7 @@ env: CHECKOUT_REF: "main" DOCKERHUB_REPOSITORY: "erigontech/erigon" LABEL_DESCRIPTION: "[docker image built on a last commit id from the main branch] Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default." + KEEP_IMAGES: 100 on: push: @@ -127,7 +128,7 @@ jobs: --push \ --platform linux/amd64,linux/arm64 . - - name: export and print docker build tag + - name: export and print docker build tag, cleanup old docker images id: built_tag_export env: BUILD_VERSION: "main-${{ steps.getCommitId.outputs.short_commit_id }}" @@ -136,6 +137,36 @@ jobs: echo The following docker images have been published: echo "${{ env.DOCKERHUB_REPOSITORY }}:main-${{ env.BUILD_VERSION }}" echo "${{ env.DOCKERHUB_REPOSITORY }}:main-latest" + echo + echo "Cleanup old docker images matching pattern tag ~= main-XXXXXXX" + curl_cmd="curl -s -H \"Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}\" " + dockerhub_url='https://hub.docker.com/v2/namespaces/erigontech/repositories/erigon' + my_list () { + # First page: + next_page="$dockerhub_url/tags?page=1&page_size=100" + while [ "$next_page" != "null" ] + do + # Print tags and push dates for tags matching "main-": + $curl_cmd $next_page | jq -r '.results|.[]|.name + " " + .tag_last_pushed' | grep 'main-' + next_page=`$curl_cmd $next_page | jq '.next' | sed -e 's/^\"//' -e 's/\"$//'` + done + } + + my_list | tail -n+${{ env.KEEP_IMAGES }} | while read line; do + echo -n "Removing docker image/published - $line " + current_image=$(echo $line | sed -e 's/^\(main-.\{7\}\) .*/\1/') + output_code=$(curl --write-out %{http_code} --output curl-output.log \ + -s -X DELETE -H "Accept: application/json" \ + -H "Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}" \ + https://hub.docker.com/v2/repositories/erigontech/erigon/tags/${current_image} ) + if [ $output_code -ne 204 ]; then + echo "ERROR: failed to remove docker image erigon:${current_image}" + echo "ERROR: API response: $(cat curl-output.log)." + else + echo -n " - removed. " + fi + echo "Done." + done run-kurtosis-assertoor: needs: [define_matrix, Build] @@ -143,4 +174,4 @@ jobs: with: checkout_ref: ${{ github.sha }} os: ${{ needs.define_matrix.outputs.os }} - docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }} \ No newline at end of file + docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }} From 063147285ea282451d9098bcf14275f735042c68 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Wed, 6 Nov 2024 10:24:27 +0000 Subject: [PATCH 7/9] Strongerrunner (#12647) Updated test names --- .github/workflows/test-erigon-is-library.yml | 2 +- .github/workflows/test-integration.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-erigon-is-library.yml b/.github/workflows/test-erigon-is-library.yml index e165dca93b7..50a20f0e224 100644 --- a/.github/workflows/test-erigon-is-library.yml +++ b/.github/workflows/test-erigon-is-library.yml @@ -1,4 +1,4 @@ -name: Integration tests +name: Library integration tests on: push: branches: diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index f47c4b0dcaa..b90dff0d477 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -1,4 +1,4 @@ -name: Integration tests +name: Erigon integration tests on: push: branches: From 82af95424f3b699de4a889428c7be5ab1c6e2e55 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Wed, 6 Nov 2024 12:05:30 +0000 Subject: [PATCH 8/9] revert changes (#12649) Revert parallel test execution cause gitlab runner is not strong enough (https://github.com/erigontech/erigon/issues/12644) --- .github/workflows/test-erigon-is-library.yml | 2 +- .github/workflows/test-integration.yml | 5 ++--- Makefile | 2 +- tests/block_test.go | 1 - tests/exec_spec_test.go | 1 - tests/state_test.go | 1 - 6 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-erigon-is-library.yml b/.github/workflows/test-erigon-is-library.yml index 50a20f0e224..e165dca93b7 100644 --- a/.github/workflows/test-erigon-is-library.yml +++ b/.github/workflows/test-erigon-is-library.yml @@ -1,4 +1,4 @@ -name: Library integration tests +name: Integration tests on: push: branches: diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index b90dff0d477..c7462a8d262 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -1,4 +1,4 @@ -name: Erigon integration tests +name: Integration tests on: push: branches: @@ -24,7 +24,6 @@ jobs: os: - ubuntu-22.04 - macos-14 - - ubuntu-latest-erigontests-large runs-on: ${{ matrix.os }} steps: @@ -38,7 +37,7 @@ jobs: run: sudo apt update && sudo apt install build-essential - name: test-integration - run: GOGC=50 make test-integration + run: make test-integration tests-windows: strategy: diff --git a/Makefile b/Makefile index 55e035f8362..aab922d9e06 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE} GOBUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build $(GO_FLAGS) GO_DBG_BUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l" # see delve docs -GOTEST = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" GODEBUG=cgocheck=0 GOTRACEBACK=1 $(GO) test $(GO_FLAGS) ./... +GOTEST = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" GODEBUG=cgocheck=0 GOTRACEBACK=1 $(GO) test $(GO_FLAGS) ./... -p 2 default: all diff --git a/tests/block_test.go b/tests/block_test.go index 3003fd707a0..269dde4ab78 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -53,7 +53,6 @@ func TestBlockchain(t *testing.T) { checkStateRoot := true bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - t.Parallel() // import pre accounts & construct test genesis block & state root if err := bt.checkFailure(t, test.Run(t, checkStateRoot)); err != nil { t.Error(err) diff --git a/tests/exec_spec_test.go b/tests/exec_spec_test.go index e93e660f97b..d8ea375f79a 100644 --- a/tests/exec_spec_test.go +++ b/tests/exec_spec_test.go @@ -35,7 +35,6 @@ func TestExecutionSpec(t *testing.T) { checkStateRoot := true bt.walk(t, dir, func(t *testing.T, name string, test *BlockTest) { - t.Parallel() // import pre accounts & construct test genesis block & state root if err := bt.checkFailure(t, test.Run(t, checkStateRoot)); err != nil { t.Error(err) diff --git a/tests/state_test.go b/tests/state_test.go index 7199c444aac..9b308a99a57 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -39,7 +39,6 @@ import ( ) func TestState(t *testing.T) { - t.Parallel() defer log.Root().SetHandler(log.Root().GetHandler()) log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StderrHandler)) if runtime.GOOS == "windows" { From fb6f8f42f4b7521f8f5225477a04336af8365294 Mon Sep 17 00:00:00 2001 From: Michelangelo Riccobene Date: Wed, 6 Nov 2024 18:46:31 +0100 Subject: [PATCH 9/9] qa-tests: use external shell script in rpc tests (#12652) Using a shell script to start the python test is useful - to manually launch tests on debug machines and - to allow us to insert comments that would otherwise violate yaml rules into the workflow --- .../workflows/qa-rpc-integration-tests.yml | 46 +-------- .github/workflows/scripts/run_rpc_tests.sh | 98 +++++++++++++++++++ 2 files changed, 101 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/scripts/run_rpc_tests.sh diff --git a/.github/workflows/qa-rpc-integration-tests.yml b/.github/workflows/qa-rpc-integration-tests.yml index bc6def09f5a..74cf9e0aca1 100644 --- a/.github/workflows/qa-rpc-integration-tests.yml +++ b/.github/workflows/qa-rpc-integration-tests.yml @@ -84,50 +84,10 @@ jobs: cd ${{ runner.workspace }}/rpc-tests/integration rm -rf ./mainnet/results/ - + # Run RPC integration test runner via http - python3 ./run_tests.py -p 8545 --continue -f --json-diff -x \ -# Erigon2 and Erigon3 never supported this api methods - trace_rawTransaction,\ -# false positives: Erigon return expected response. but rpc-test somehow doesn't see 1 field. - erigon_getHeaderByHash,erigon_getHeaderByNumber,eth_feeHistory,\ -# total difficulty field was removed, then added back - eth_getBlockByHash,eth_getBlockByNumber,\ -# Erigon bugs - debug_accountRange,debug_storageRangeAt,\ -# need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634) - debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber,\ -# Erigon bug https://github.com/erigontech/erigon/issues/12603 - erigon_getLatestLogs,erigon_getLogsByHash/test_04.json,\ -# Erigon bug https://github.com/erigontech/erigon/issues/12637 - debug_traceBlockByNumber/test_05.tar,debug_traceBlockByNumber/test_08.tar,debug_traceBlockByNumber/test_09.tar,debug_traceBlockByNumber/test_10.tar,debug_traceBlockByNumber/test_11.tar,debug_traceBlockByNumber/test_12.tar,\ -# remove this line after https://github.com/erigontech/rpc-tests/pull/281 - parity_getBlockReceipts,\ -# to investigate - debug_traceBlockByHash,\ - debug_traceCallMany/test_02.tar,debug_traceCallMany/test_04.tar,debug_traceCallMany/test_05.tar,debug_traceCallMany/test_06.tar,debug_traceCallMany/test_07.tar,debug_traceCallMany/test_09.json,debug_traceCallMany/test_10.tar,\ - engine_exchangeCapabilities/test_1.json,\ - engine_exchangeTransitionConfigurationV1/test_01.json,\ - engine_getClientVersionV1/test_1.json,\ - erigon_getBalanceChangesInBlock,\ - eth_createAccessList/test_16.json,\ - admin_nodeInfo/test_01.json,\ - admin_peers/test_01.json,\ - erigon_nodeInfo/test_1.json,\ - eth_coinbase/test_01.json,\ - eth_getTransactionByHash/test_02.json,\ - eth_getWork/test_01.json,\ - eth_mining/test_01.json,\ - eth_protocolVersion/test_1.json,\ - eth_submitHashrate/test_1.json,\ - eth_submitWork/test_1.json,\ - net_peerCount/test_1.json,\ - net_version/test_1.json,\ - txpool_content/test_01.json,\ - txpool_status/test_1.json,\ - web3_clientVersion/test_1.json,\ - eth_estimateGas/test_14.json,\ - trace_replayBlockTransactions/test_29.tar + chmod +x ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh + ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh # Capture test runner script exit status test_exit_status=$? diff --git a/.github/workflows/scripts/run_rpc_tests.sh b/.github/workflows/scripts/run_rpc_tests.sh new file mode 100644 index 00000000000..0f94f68c039 --- /dev/null +++ b/.github/workflows/scripts/run_rpc_tests.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set +e # Disable exit on error + +# Array of disabled tests +disabled_tests=( + # Erigon2 and Erigon3 never supported this api methods + trace_rawTransaction + # false positives: Erigon return expected response. but rpc-test somehow doesn't see 1 field. + erigon_getHeaderByHash,erigon_getHeaderByNumber,eth_feeHistory + # total difficulty field was removed, then added back + eth_getBlockByHash,eth_getBlockByNumber + # Erigon bugs + debug_accountRange,debug_storageRangeAt + # need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634) + debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber + # Erigon bug https://github.com/erigontech/erigon/issues/12603 + erigon_getLatestLogs,erigon_getLogsByHash/test_04.json + # Erigon bug https://github.com/erigontech/erigon/issues/12637 + debug_traceBlockByNumber/test_05.tar + debug_traceBlockByNumber/test_08.tar + debug_traceBlockByNumber/test_09.tar + debug_traceBlockByNumber/test_10.tar + debug_traceBlockByNumber/test_11.tar + debug_traceBlockByNumber/test_12.tar + # remove this line after https://github.com/erigontech/rpc-tests/pull/281 + parity_getBlockReceipts + # to investigate + debug_traceBlockByHash + debug_traceCallMany/test_02.tar + debug_traceCallMany/test_04.tar + debug_traceCallMany/test_05.tar + debug_traceCallMany/test_06.tar + debug_traceCallMany/test_07.tar + debug_traceCallMany/test_09.json + debug_traceCallMany/test_10.tar + engine_exchangeCapabilities/test_1.json + engine_exchangeTransitionConfigurationV1/test_01.json + engine_getClientVersionV1/test_1.json + erigon_getBalanceChangesInBlock + eth_createAccessList/test_16.json + admin_nodeInfo/test_01.json + admin_peers/test_01.json + erigon_nodeInfo/test_1.json + eth_coinbase/test_01.json + eth_getTransactionByHash/test_02.json + eth_getWork/test_01.json + eth_mining/test_01.json + eth_protocolVersion/test_1.json + eth_submitHashrate/test_1.json + eth_submitWork/test_1.json + net_peerCount/test_1.json + net_version/test_1.json + txpool_content/test_01.json + txpool_status/test_1.json + web3_clientVersion/test_1.json + eth_estimateGas/test_14.json + trace_replayBlockTransactions/test_29.tar + # recently started to fail + debug_traceTransaction/test_20.json + debug_traceTransaction/test_21.json + debug_traceTransaction/test_22.json + debug_traceTransaction/test_25.json + debug_traceTransaction/test_30.tar + debug_traceTransaction/test_33.json + debug_traceTransaction/test_35.tar + debug_traceTransaction/test_36.json + debug_traceTransaction/test_37.tar + debug_traceTransaction/test_38.tar + debug_traceTransaction/test_43.json + debug_traceTransaction/test_44.json + debug_traceTransaction/test_62.json + debug_traceTransaction/test_64.json + debug_traceTransaction/test_74.tar + debug_traceTransaction/test_75.tar + debug_traceTransaction/test_77.json + debug_traceTransaction/test_78.tar + debug_traceTransaction/test_79.tar + debug_traceTransaction/test_80.tar + debug_traceTransaction/test_81.tar + debug_traceTransaction/test_82.tar + debug_traceTransaction/test_83.tar + debug_traceTransaction/test_84.tar + debug_traceTransaction/test_85.tar + debug_traceTransaction/test_87.json + debug_traceTransaction/test_90.tar + debug_traceTransaction/test_91.tar + debug_traceTransaction/test_92.tar + debug_traceTransaction/test_93.json + debug_traceTransaction/test_96.json + trace_filter/test_16.json) + +# Transform the array into a comma-separated string +disabled_test_list=$(IFS=,; echo "${disabled_tests[*]}") + +python3 ./run_tests.py -p 8545 --continue -f --json-diff -x "$disabled_test_list" + +exit $? \ No newline at end of file