diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go index 3cdd3de87cc..a85787e8e99 100644 --- a/cl/antiquary/state_antiquary.go +++ b/cl/antiquary/state_antiquary.go @@ -23,6 +23,7 @@ import ( "github.com/ledgerwatch/erigon/cl/persistence/base_encoding" "github.com/ledgerwatch/erigon/cl/persistence/beacon_indicies" state_accessors "github.com/ledgerwatch/erigon/cl/persistence/state" + "github.com/ledgerwatch/erigon/cl/persistence/state/historical_states_reader" "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/phase1/core/state/raw" "github.com/ledgerwatch/erigon/cl/phase1/core/state/shuffling" @@ -134,6 +135,7 @@ func uint64BalancesList(s *state.CachingBeaconState, out []uint64) []uint64 { func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { var tx kv.Tx + tx, err := s.mainDB.BeginRo(ctx) if err != nil { return err @@ -182,6 +184,20 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { stateEvents := etl.NewCollector(kv.StateEvents, s.dirs.Tmp, etl.NewSortableBuffer(etl.BufferOptimalSize), s.logger) defer stateEvents.Close() + progress, err := state_accessors.GetStateProcessingProgress(tx) + if err != nil { + return err + } + // Go back a little bit + if progress > s.cfg.SlotsPerEpoch*2 { + progress -= s.cfg.SlotsPerEpoch * 2 + } else { + progress = 0 + } + progress, err = findNearestSlotBackwards(tx, progress) // Maybe the guess was a missed slot. + if err != nil { + return err + } // buffers commonBuffer := &bytes.Buffer{} compressedWriter, err := zstd.NewWriter(commonBuffer, zstd.WithEncoderLevel(zstd.SpeedBetterCompression)) @@ -189,18 +205,35 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { return err } defer compressedWriter.Close() - // TODO(Giulio2002): also store genesis information and resume from state. if s.currentState == nil { - s.currentState, err = s.genesisState.Copy() - if err != nil { - return err - } - // Collect genesis state if we are at genesis - if err := s.collectGenesisState(ctx, compressedWriter, s.currentState, slashings, proposers, minimalBeaconStates, stateEvents, changedValidators); err != nil { - return err + // progress is 0 when we are at genesis + if progress == 0 { + s.currentState, err = s.genesisState.Copy() + if err != nil { + return err + } + // Collect genesis state if we are at genesis + if err := s.collectGenesisState(ctx, compressedWriter, s.currentState, slashings, proposers, minimalBeaconStates, stateEvents, changedValidators); err != nil { + return err + } + } else { + start := time.Now() + // progress not 0? we need to load the state from the DB + historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.fs, s.genesisState) + s.currentState, err = historicalReader.ReadHistoricalState(ctx, tx, progress) + if err != nil { + return fmt.Errorf("failed to read historical state at slot %d: %w", progress, err) + } + end := time.Since(start) + hashRoot, err := s.currentState.HashSSZ() + if err != nil { + return err + } + log.Info("Recovered Beacon State", "slot", s.currentState.Slot(), "elapsed", end, "root", libcommon.Hash(hashRoot).String()) } } + logLvl := log.LvlInfo if to-s.currentState.Slot() < 96 { logLvl = log.LvlDebug @@ -256,8 +289,11 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { return s.validatorsTable.AddWithdrawalCredentials(uint64(index), slot, libcommon.BytesToHash(wc)) }, OnEpochBoundary: func(epoch uint64) error { - v := append(s.currentState.CurrentJustifiedCheckpoint(), append(s.currentState.PreviousJustifiedCheckpoint(), s.currentState.FinalizedCheckpoint()...)...) k := base_encoding.Encode64ToBytes4(s.cfg.RoundSlotToEpoch(slot)) + v := make([]byte, solid.CheckpointSize*3) + copy(v, s.currentState.CurrentJustifiedCheckpoint()) + copy(v[solid.CheckpointSize:], s.currentState.PreviousJustifiedCheckpoint()) + copy(v[solid.CheckpointSize*2:], s.currentState.FinalizedCheckpoint()) if err := checkpoints.Collect(k, v); err != nil { return err } @@ -302,9 +338,11 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { progressTimer := time.NewTicker(1 * time.Minute) defer progressTimer.Stop() prevSlot := slot + first := false // This tells us that transition and operations do not happen concurrently and access is safe, so we can optimize for GC. // there is optimized custom cache to recycle big GC overhead. for ; slot < to; slot++ { + isDumpSlot := slot%clparams.SlotsPerDump == 0 block, err := s.snReader.ReadBlockBySlot(ctx, tx, slot) if err != nil { return err @@ -329,11 +367,11 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { } } - if slot%clparams.SlotsPerDump == 0 { + if isDumpSlot && block == nil { if err := s.antiquateField(ctx, slot, s.currentState.RawBalances(), compressedWriter, "balances"); err != nil { return err } - if err := s.antiquateEffectiveBalances(ctx, slot, s.currentState.RawBalances(), compressedWriter); err != nil { + if err := s.antiquateEffectiveBalances(ctx, slot, s.currentState.RawValidatorSet(), compressedWriter); err != nil { return err } if s.currentState.Version() >= clparams.AltairVersion { @@ -360,11 +398,15 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { slashingsBytes = slashingsBytes[:0] slashingsBytes = append(slashingsBytes, s.currentState.RawSlashings()...) - // We sanity check the state every 100k slots. - if err := transition.TransitionState(s.currentState, block, slot%100_000 == 0); err != nil { + fullValidation := slot%100_000 == 0 || first + // We sanity check the state every 100k slots or when we start. + if err := transition.TransitionState(s.currentState, block, fullValidation); err != nil { return err } - + first = false + // if s.currentState.Slot() == 3868670 { + // s.dumpFullBeaconState() + // } if err := s.storeMinimalState(commonBuffer, s.currentState, minimalBeaconStates); err != nil { return err } @@ -372,7 +414,22 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { return err } events.Reset() - if slot%clparams.SlotsPerDump == 0 { + + if isDumpSlot { + if err := s.antiquateField(ctx, slot, s.currentState.RawBalances(), compressedWriter, "balances"); err != nil { + return err + } + if err := s.antiquateEffectiveBalances(ctx, slot, s.currentState.RawValidatorSet(), compressedWriter); err != nil { + return err + } + if s.currentState.Version() >= clparams.AltairVersion { + if err := s.antiquateField(ctx, slot, s.currentState.RawInactivityScores(), compressedWriter, "inactivity_scores"); err != nil { + return err + } + } + if err := s.antiquateFullSlashings(slashings, slot, s.currentState.RawSlashings(), commonBuffer, compressedWriter); err != nil { + return err + } continue } @@ -503,9 +560,16 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error { if err != nil { return err } - - log.Info("Historical antiquated", "slot", s.currentState.Slot(), "latency", time.Since(start)) - return rwTx.Commit() + if err := rwTx.Commit(); err != nil { + return err + } + endTime := time.Since(start) + stateRoot, err := s.currentState.HashSSZ() + if err != nil { + return err + } + log.Info("Historical states antiquated", "slot", s.currentState.Slot(), "root", libcommon.Hash(stateRoot), "latency", endTime) + return nil } func (s *Antiquary) antiquateField(ctx context.Context, slot uint64, uncompressed []byte, compressor *zstd.Encoder, name string) error { @@ -700,6 +764,19 @@ func (s *Antiquary) dumpPayload(k []byte, v []byte, c *etl.Collector, b *bytes.B return c.Collect(k, common.Copy(b.Bytes())) } +// func (s *Antiquary) dumpFullBeaconState() { +// b, err := s.currentState.EncodeSSZ(nil) +// if err != nil { +// s.logger.Error("Failed to encode full beacon state", "err", err) +// return +// } +// // just dump it in a.txt like an idiot without afero +// if err := os.WriteFile("b.txt", b, 0644); err != nil { +// s.logger.Error("Failed to write full beacon state", "err", err) +// } + +// } + func flattenRandaoMixes(hashes []libcommon.Hash) []byte { out := make([]byte, len(hashes)*32) for i, h := range hashes { @@ -720,3 +797,18 @@ func (s *Antiquary) antiquateFullSlashings(collector *etl.Collector, slot uint64 } return collector.Collect(base_encoding.Encode64ToBytes4(slot), common.Copy(buffer.Bytes())) } + +func findNearestSlotBackwards(tx kv.Tx, slot uint64) (uint64, error) { + canonicalRoot, err := beacon_indicies.ReadCanonicalBlockRoot(tx, slot) + if err != nil { + return 0, err + } + for canonicalRoot == (common.Hash{}) && slot > 0 { + slot-- + canonicalRoot, err = beacon_indicies.ReadCanonicalBlockRoot(tx, slot) + if err != nil { + return 0, err + } + } + return slot, nil +} diff --git a/cl/antiquary/state_antiquary_test.go b/cl/antiquary/state_antiquary_test.go index ea24cf7049b..73688024e0f 100644 --- a/cl/antiquary/state_antiquary_test.go +++ b/cl/antiquary/state_antiquary_test.go @@ -2,6 +2,7 @@ package antiquary import ( "context" + "fmt" "testing" _ "embed" @@ -36,6 +37,13 @@ func TestStateAntiquaryCapella(t *testing.T) { runTest(t, blocks, preState, postState) } +func TestStateAntiquaryBellatrix(t *testing.T) { + t.Skip() + blocks, preState, postState := tests.GetBellatrixRandom() + fmt.Println(len(blocks)) + runTest(t, blocks, preState, postState) +} + func TestStateAntiquaryPhase0(t *testing.T) { t.Skip() blocks, preState, postState := tests.GetPhase0Random() diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_0.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_0.ssz_snappy new file mode 100644 index 00000000000..ddb86265417 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_0.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_1.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_1.ssz_snappy new file mode 100644 index 00000000000..338d0aa9f5c Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_1.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_10.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_10.ssz_snappy new file mode 100644 index 00000000000..f15263647c4 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_10.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_11.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_11.ssz_snappy new file mode 100644 index 00000000000..2480b5d7ed3 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_11.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_12.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_12.ssz_snappy new file mode 100644 index 00000000000..0ff0fabc2ad Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_12.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_13.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_13.ssz_snappy new file mode 100644 index 00000000000..802619428d7 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_13.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_14.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_14.ssz_snappy new file mode 100644 index 00000000000..e7f70750845 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_14.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_15.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_15.ssz_snappy new file mode 100644 index 00000000000..b9d5ef35308 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_15.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_16.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_16.ssz_snappy new file mode 100644 index 00000000000..29d509c9a61 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_16.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_17.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_17.ssz_snappy new file mode 100644 index 00000000000..3f976bd89e9 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_17.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_18.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_18.ssz_snappy new file mode 100644 index 00000000000..39f66b4d87f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_18.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_19.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_19.ssz_snappy new file mode 100644 index 00000000000..5ee1b7ae12b Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_19.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_2.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_2.ssz_snappy new file mode 100644 index 00000000000..e5d9c477572 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_2.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_20.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_20.ssz_snappy new file mode 100644 index 00000000000..09979630358 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_20.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_21.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_21.ssz_snappy new file mode 100644 index 00000000000..b611fe260ee Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_21.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_22.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_22.ssz_snappy new file mode 100644 index 00000000000..7f97dafd851 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_22.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_23.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_23.ssz_snappy new file mode 100644 index 00000000000..74f348df968 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_23.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_24.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_24.ssz_snappy new file mode 100644 index 00000000000..c0a4e649b1c Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_24.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_25.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_25.ssz_snappy new file mode 100644 index 00000000000..83ec1717069 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_25.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_26.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_26.ssz_snappy new file mode 100644 index 00000000000..4299bb3cd17 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_26.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_27.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_27.ssz_snappy new file mode 100644 index 00000000000..6635bb8ae6d Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_27.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_28.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_28.ssz_snappy new file mode 100644 index 00000000000..5b31157fd7b Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_28.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_29.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_29.ssz_snappy new file mode 100644 index 00000000000..b9cb2b91feb Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_29.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_3.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_3.ssz_snappy new file mode 100644 index 00000000000..93448ab0b95 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_3.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_30.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_30.ssz_snappy new file mode 100644 index 00000000000..ef1c4a4f625 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_30.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_31.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_31.ssz_snappy new file mode 100644 index 00000000000..aaf877bb3ec Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_31.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_32.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_32.ssz_snappy new file mode 100644 index 00000000000..ee19e4ef320 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_32.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_33.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_33.ssz_snappy new file mode 100644 index 00000000000..cb6b0514d2f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_33.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_34.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_34.ssz_snappy new file mode 100644 index 00000000000..9f7d82de01f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_34.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_35.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_35.ssz_snappy new file mode 100644 index 00000000000..942d790bcab Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_35.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_36.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_36.ssz_snappy new file mode 100644 index 00000000000..fe7bfd23487 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_36.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_37.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_37.ssz_snappy new file mode 100644 index 00000000000..3de6efbf78e Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_37.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_38.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_38.ssz_snappy new file mode 100644 index 00000000000..b79724bba01 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_38.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_39.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_39.ssz_snappy new file mode 100644 index 00000000000..9cbebe9fef2 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_39.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_4.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_4.ssz_snappy new file mode 100644 index 00000000000..ff1c6ebb6a9 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_4.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_40.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_40.ssz_snappy new file mode 100644 index 00000000000..7a63b458fa0 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_40.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_41.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_41.ssz_snappy new file mode 100644 index 00000000000..77fc5a1f5b9 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_41.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_42.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_42.ssz_snappy new file mode 100644 index 00000000000..7231136bb3b Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_42.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_43.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_43.ssz_snappy new file mode 100644 index 00000000000..3e8eb3d2aea Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_43.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_44.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_44.ssz_snappy new file mode 100644 index 00000000000..fb6af616f2e Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_44.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_45.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_45.ssz_snappy new file mode 100644 index 00000000000..fe544a77ae1 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_45.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_46.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_46.ssz_snappy new file mode 100644 index 00000000000..9455c0c00bc Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_46.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_47.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_47.ssz_snappy new file mode 100644 index 00000000000..998b1cdd4ab Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_47.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_48.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_48.ssz_snappy new file mode 100644 index 00000000000..95dc1add11a Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_48.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_49.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_49.ssz_snappy new file mode 100644 index 00000000000..b1b9f4a5320 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_49.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_5.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_5.ssz_snappy new file mode 100644 index 00000000000..70322b97f8f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_5.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_50.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_50.ssz_snappy new file mode 100644 index 00000000000..d1878140277 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_50.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_51.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_51.ssz_snappy new file mode 100644 index 00000000000..bb247b4edcc Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_51.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_52.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_52.ssz_snappy new file mode 100644 index 00000000000..4591a85f7ca Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_52.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_53.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_53.ssz_snappy new file mode 100644 index 00000000000..16ae7cf07e6 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_53.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_54.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_54.ssz_snappy new file mode 100644 index 00000000000..ce039121c6d Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_54.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_55.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_55.ssz_snappy new file mode 100644 index 00000000000..cfd569604fb Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_55.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_56.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_56.ssz_snappy new file mode 100644 index 00000000000..c3ff7830aae Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_56.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_57.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_57.ssz_snappy new file mode 100644 index 00000000000..a96e3bc83d5 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_57.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_58.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_58.ssz_snappy new file mode 100644 index 00000000000..e377414e3f8 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_58.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_59.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_59.ssz_snappy new file mode 100644 index 00000000000..30eb999215f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_59.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_6.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_6.ssz_snappy new file mode 100644 index 00000000000..443eb6f6852 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_6.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_60.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_60.ssz_snappy new file mode 100644 index 00000000000..c5abd4e7052 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_60.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_61.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_61.ssz_snappy new file mode 100644 index 00000000000..15f0c0eba5e Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_61.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_62.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_62.ssz_snappy new file mode 100644 index 00000000000..6329d0b0b7f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_62.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_63.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_63.ssz_snappy new file mode 100644 index 00000000000..9a788865bc8 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_63.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_64.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_64.ssz_snappy new file mode 100644 index 00000000000..6211aef59c9 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_64.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_65.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_65.ssz_snappy new file mode 100644 index 00000000000..9f9fe354302 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_65.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_66.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_66.ssz_snappy new file mode 100644 index 00000000000..2d8d6410aa1 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_66.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_67.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_67.ssz_snappy new file mode 100644 index 00000000000..a7d59ebd37d Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_67.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_68.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_68.ssz_snappy new file mode 100644 index 00000000000..ba24c242ba0 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_68.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_69.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_69.ssz_snappy new file mode 100644 index 00000000000..d0ee1bd34d3 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_69.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_7.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_7.ssz_snappy new file mode 100644 index 00000000000..d0270aab04d Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_7.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_70.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_70.ssz_snappy new file mode 100644 index 00000000000..589aaa2bbe3 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_70.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_71.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_71.ssz_snappy new file mode 100644 index 00000000000..0c0c3dac0c3 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_71.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_72.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_72.ssz_snappy new file mode 100644 index 00000000000..31600d4849b Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_72.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_73.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_73.ssz_snappy new file mode 100644 index 00000000000..5e2b0eea6af Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_73.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_74.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_74.ssz_snappy new file mode 100644 index 00000000000..08404e98615 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_74.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_75.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_75.ssz_snappy new file mode 100644 index 00000000000..31260532e91 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_75.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_76.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_76.ssz_snappy new file mode 100644 index 00000000000..6f8e00d1a53 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_76.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_77.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_77.ssz_snappy new file mode 100644 index 00000000000..cc9c4c6f043 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_77.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_78.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_78.ssz_snappy new file mode 100644 index 00000000000..ab1a71ce711 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_78.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_79.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_79.ssz_snappy new file mode 100644 index 00000000000..5c0aa21445d Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_79.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_8.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_8.ssz_snappy new file mode 100644 index 00000000000..13fa263a79f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_8.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_80.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_80.ssz_snappy new file mode 100644 index 00000000000..12f4c18084c Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_80.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_81.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_81.ssz_snappy new file mode 100644 index 00000000000..0625f98e73c Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_81.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_82.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_82.ssz_snappy new file mode 100644 index 00000000000..a73e8ba3abb Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_82.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_83.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_83.ssz_snappy new file mode 100644 index 00000000000..c85e4aa1598 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_83.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_84.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_84.ssz_snappy new file mode 100644 index 00000000000..a476b0929b6 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_84.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_85.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_85.ssz_snappy new file mode 100644 index 00000000000..f097123ef54 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_85.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_86.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_86.ssz_snappy new file mode 100644 index 00000000000..7b3eddf3f8c Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_86.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_87.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_87.ssz_snappy new file mode 100644 index 00000000000..638bafe5d61 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_87.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_88.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_88.ssz_snappy new file mode 100644 index 00000000000..ef553a34db2 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_88.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_89.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_89.ssz_snappy new file mode 100644 index 00000000000..1591667ac26 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_89.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_9.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_9.ssz_snappy new file mode 100644 index 00000000000..b4c3dcfd693 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_9.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_90.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_90.ssz_snappy new file mode 100644 index 00000000000..373746aa808 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_90.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_91.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_91.ssz_snappy new file mode 100644 index 00000000000..fc01404a9c6 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_91.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_92.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_92.ssz_snappy new file mode 100644 index 00000000000..cd851715494 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_92.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_93.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_93.ssz_snappy new file mode 100644 index 00000000000..636ade4791f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_93.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_94.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_94.ssz_snappy new file mode 100644 index 00000000000..b07de428a13 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_94.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/blocks_95.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/blocks_95.ssz_snappy new file mode 100644 index 00000000000..7cbf6df9079 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/blocks_95.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/meta.yaml b/cl/antiquary/tests/test_data/bellatrix/meta.yaml new file mode 100644 index 00000000000..46caff2710c --- /dev/null +++ b/cl/antiquary/tests/test_data/bellatrix/meta.yaml @@ -0,0 +1 @@ +{blocks_count: 96} diff --git a/cl/antiquary/tests/test_data/bellatrix/post.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/post.ssz_snappy new file mode 100644 index 00000000000..b3bc26c91c2 Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/post.ssz_snappy differ diff --git a/cl/antiquary/tests/test_data/bellatrix/pre.ssz_snappy b/cl/antiquary/tests/test_data/bellatrix/pre.ssz_snappy new file mode 100644 index 00000000000..ee9a847b53f Binary files /dev/null and b/cl/antiquary/tests/test_data/bellatrix/pre.ssz_snappy differ diff --git a/cl/antiquary/tests/tests.go b/cl/antiquary/tests/tests.go index 98ee2ff043b..1596e16d05b 100644 --- a/cl/antiquary/tests/tests.go +++ b/cl/antiquary/tests/tests.go @@ -2,7 +2,9 @@ package tests import ( "context" + "embed" _ "embed" + "strconv" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" @@ -37,6 +39,11 @@ var phase0_pre_state_ssz_snappy []byte //go:embed test_data/phase0/post.ssz_snappy var phase0_post_state_ssz_snappy []byte +// bellatrix is long + +//go:embed test_data/bellatrix +var bellatrixFS embed.FS + type MockBlockReader struct { u map[uint64]*cltypes.SignedBeaconBlock } @@ -130,3 +137,38 @@ func GetPhase0Random() ([]*cltypes.SignedBeaconBlock, *state.CachingBeaconState, } return []*cltypes.SignedBeaconBlock{block1, block2}, preState, postState } + +func GetBellatrixRandom() ([]*cltypes.SignedBeaconBlock, *state.CachingBeaconState, *state.CachingBeaconState) { + ret := make([]*cltypes.SignedBeaconBlock, 0, 96) + // format for blocks is blocks_{i}.ssz_snappy where i is the index of the block, starting from 0 to 95 included. + for i := 0; i < 96; i++ { + block := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig) + // Lets do te + b, err := bellatrixFS.ReadFile("test_data/bellatrix/blocks_" + strconv.FormatInt(int64(i), 10) + ".ssz_snappy") + if err != nil { + panic(err) + } + if err := utils.DecodeSSZSnappy(block, b, int(clparams.BellatrixVersion)); err != nil { + panic(err) + } + ret = append(ret, block) + } + preState := state.New(&clparams.MainnetBeaconConfig) + b, err := bellatrixFS.ReadFile("test_data/bellatrix/pre.ssz_snappy") + if err != nil { + panic(err) + } + if err := utils.DecodeSSZSnappy(preState, b, int(clparams.BellatrixVersion)); err != nil { + panic(err) + } + postState := state.New(&clparams.MainnetBeaconConfig) + b, err = bellatrixFS.ReadFile("test_data/bellatrix/post.ssz_snappy") + if err != nil { + panic(err) + } + if err := utils.DecodeSSZSnappy(postState, b, int(clparams.BellatrixVersion)); err != nil { + panic(err) + } + return ret, preState, postState + +} diff --git a/cl/cltypes/eth1_block.go b/cl/cltypes/eth1_block.go index ef5819c648c..3d92938f7ee 100644 --- a/cl/cltypes/eth1_block.go +++ b/cl/cltypes/eth1_block.go @@ -94,8 +94,11 @@ func (*Eth1Block) Static() bool { func (b *Eth1Block) PayloadHeader() (*Eth1Header, error) { var err error var transactionsRoot, withdrawalsRoot libcommon.Hash - if transactionsRoot, err = b.Transactions.HashSSZ(); err != nil { - return nil, err + // Corner case: before TTD this is 0, since all fields are 0, a 0 hash check will suffice. + if b.BlockHash != (libcommon.Hash{}) { + if transactionsRoot, err = b.Transactions.HashSSZ(); err != nil { + return nil, err + } } if b.version >= clparams.CapellaVersion { withdrawalsRoot, err = b.Withdrawals.HashSSZ() diff --git a/cl/cltypes/solid/checkpoint.go b/cl/cltypes/solid/checkpoint.go index 8db388792ce..87ce50436aa 100644 --- a/cl/cltypes/solid/checkpoint.go +++ b/cl/cltypes/solid/checkpoint.go @@ -13,7 +13,7 @@ import ( ) // Constants to represent the size and layout of a Checkpoint -const checkpointSize = 32 + 8 // BlockRoot(32 bytes) + Epoch(8 bytes) +const CheckpointSize = 32 + 8 // BlockRoot(32 bytes) + Epoch(8 bytes) type Checkpoint []byte // Define Checkpoint as a byte slice @@ -22,7 +22,7 @@ func NewCheckpointFromParameters( blockRoot libcommon.Hash, // A hash representing the block root epoch uint64, // An unsigned 64-bit integer representing the epoch ) Checkpoint { - var c Checkpoint = make([]byte, checkpointSize) + var c Checkpoint = make([]byte, CheckpointSize) c.SetBlockRoot(blockRoot) c.SetEpoch(epoch) return c @@ -30,7 +30,7 @@ func NewCheckpointFromParameters( // NewCheckpoint returns a new Checkpoint with the underlying byte slice initialized to zeros func NewCheckpoint() Checkpoint { - return make([]byte, checkpointSize) + return make([]byte, CheckpointSize) } func (c Checkpoint) MarshalJSON() ([]byte, error) { @@ -92,7 +92,7 @@ func (c Checkpoint) BlockRoot() (o libcommon.Hash) { // EncodingSizeSSZ returns the size of the Checkpoint object when encoded as SSZ. func (Checkpoint) EncodingSizeSSZ() int { - return checkpointSize + return CheckpointSize } // DecodeSSZ decodes the Checkpoint object from SSZ-encoded data. diff --git a/cl/cltypes/solid/uint64slice_byte.go b/cl/cltypes/solid/uint64slice_byte.go index ce102e81f2b..a642c6278c1 100644 --- a/cl/cltypes/solid/uint64slice_byte.go +++ b/cl/cltypes/solid/uint64slice_byte.go @@ -81,7 +81,7 @@ func (arr *byteBasedUint64Slice) CopyTo(target *byteBasedUint64Slice) { func (arr *byteBasedUint64Slice) MarshalJSON() ([]byte, error) { list := make([]uint64, arr.l) for i := 0; i < arr.l; i++ { - list[0] = arr.Get(i) + list[i] = arr.Get(i) } return json.Marshal(list) } diff --git a/cl/cltypes/solid/validator_set.go b/cl/cltypes/solid/validator_set.go index c8241fe0880..eb5b6e170d1 100644 --- a/cl/cltypes/solid/validator_set.go +++ b/cl/cltypes/solid/validator_set.go @@ -58,6 +58,8 @@ func NewValidatorSetWithLength(c int, l int) *ValidatorSet { l: l, buffer: make([]byte, l*validatorSize), treeCacheBuffer: make([]byte, getTreeCacheSize(l, validatorTreeCacheGroupLayer)*length.Hash), + phase0Data: make([]Phase0Data, l), + attesterBits: make([]byte, l), } } diff --git a/cl/persistence/base_encoding/uint64_diff.go b/cl/persistence/base_encoding/uint64_diff.go index 4c242cc9623..1e3496082e9 100644 --- a/cl/persistence/base_encoding/uint64_diff.go +++ b/cl/persistence/base_encoding/uint64_diff.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/klauspost/compress/zstd" + "github.com/ledgerwatch/erigon/cl/utils" ) // make a sync.pool of compressors (zstd) @@ -211,7 +212,7 @@ func ApplyCompressedSerializedUint64ListDiff(old, out []byte, diff []byte) ([]by temp := make([]byte, 8) currIndex := 0 for i := 0; i < int(length); i++ { - n, err := decompressor.Read(temp[:4]) + n, err := utils.ReadZSTD(decompressor, temp[:4]) if err != nil && !errors.Is(err, io.EOF) { return nil, err } @@ -219,7 +220,8 @@ func ApplyCompressedSerializedUint64ListDiff(old, out []byte, diff []byte) ([]by return nil, io.EOF } entry.count = binary.BigEndian.Uint32(temp[:4]) - n, err = decompressor.Read(temp) + + n, err = utils.ReadZSTD(decompressor, temp) if err != nil && !errors.Is(err, io.EOF) { return nil, err } diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go index 380f1992d72..1290f27fb1e 100644 --- a/cl/persistence/state/historical_states_reader/historical_states_reader.go +++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "sync" "github.com/klauspost/compress/zstd" "github.com/ledgerwatch/erigon-lib/common" @@ -18,6 +19,7 @@ import ( state_accessors "github.com/ledgerwatch/erigon/cl/persistence/state" "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/phase1/core/state/lru" + "github.com/ledgerwatch/erigon/cl/utils" "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/spf13/afero" "golang.org/x/exp/slices" @@ -106,12 +108,12 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. ret.SetLatestBlockHeader(blockHeader) if err := r.readHistoryHashVector(tx, r.genesisState.BlockRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.BlockRoot, blockRoots); err != nil { - return nil, err + return nil, fmt.Errorf("failed to read block roots: %w", err) } ret.SetBlockRoots(blockRoots) if err := r.readHistoryHashVector(tx, r.genesisState.StateRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.StateRoot, stateRoots); err != nil { - return nil, err + return nil, fmt.Errorf("failed to read state roots: %w", err) } ret.SetStateRoots(stateRoots) @@ -120,13 +122,13 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. historicalRoots.Append(root) return nil }); err != nil { - return nil, err + return nil, fmt.Errorf("failed to read historical roots: %w", err) } ret.SetHistoricalRoots(historicalRoots) // Eth1 - eth1DataVotes := solid.NewDynamicListSSZ[*cltypes.Eth1Data](int(r.cfg.Eth1DataVotesLength())) - if err := r.readEth1DataVotes(tx, slot, eth1DataVotes); err != nil { + eth1DataVotes := solid.NewStaticListSSZ[*cltypes.Eth1Data](int(r.cfg.Eth1DataVotesLength()), 72) + if err := r.readEth1DataVotes(tx, minimalBeaconState.Eth1DataLength, slot, eth1DataVotes); err != nil { return nil, err } ret.SetEth1DataVotes(eth1DataVotes) @@ -135,30 +137,29 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. // Registry (Validators + Balances) balancesBytes, err := r.reconstructDiffedUint64List(tx, slot, kv.ValidatorBalance, "balances") if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read validator balances: %w", err) } balances := solid.NewUint64ListSSZ(int(r.cfg.ValidatorRegistryLimit)) if err := balances.DecodeSSZ(balancesBytes, 0); err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode validator balances: %w", err) } ret.SetBalances(balances) validatorSet, currActiveIdxs, prevActiveIdxs, err := r.readValidatorsForHistoricalState(tx, slot, minimalBeaconState.ValidatorLength) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read validators: %w", err) } ret.SetValidators(validatorSet) - // Randomness randaoMixes := solid.NewHashVector(int(r.cfg.EpochsPerHistoricalVector)) if err := r.readRandaoMixes(tx, slot, randaoMixes); err != nil { - return nil, err + return nil, fmt.Errorf("failed to read randao mixes: %w", err) } ret.SetRandaoMixes(randaoMixes) // Slashings slashings, err := r.reconstructDiffedUint64Vector(tx, slot, kv.ValidatorSlashings, int(r.cfg.EpochsPerSlashingsVector)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read slashings: %w", err) } ret.SetSlashings(slashings) @@ -180,43 +181,41 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. ret.SetPreviousJustifiedCheckpoint(previousCheckpoint) ret.SetCurrentJustifiedCheckpoint(currentCheckpoint) ret.SetFinalizedCheckpoint(finalizedCheckpoint) - // Participation if ret.Version() == clparams.Phase0Version { currentAtts, previousAtts, err := r.readPendingEpochs(tx, slot, minimalBeaconState.CurrentEpochAttestationsLength, minimalBeaconState.PreviousEpochAttestationsLength) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read pending attestations: %w", err) } ret.SetCurrentEpochAttestations(currentAtts) ret.SetPreviousEpochAttestations(previousAtts) } else { currentIdxs, previousIdxs, err := r.readPartecipations(tx, slot, minimalBeaconState.ValidatorLength, currActiveIdxs, prevActiveIdxs, ret, currentCheckpoint, previousCheckpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read participations: %w", err) } ret.SetCurrentEpochParticipation(currentIdxs) ret.SetPreviousEpochParticipation(previousIdxs) } if ret.Version() < clparams.AltairVersion { - return ret, nil + return ret, ret.InitBeaconState() } // Inactivity inactivityScoresBytes, err := r.reconstructDiffedUint64List(tx, slot, kv.InactivityScores, "inactivity_scores") if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read inactivity scores: %w", err) } inactivityScores := solid.NewUint64ListSSZ(int(r.cfg.ValidatorRegistryLimit)) if err := inactivityScores.DecodeSSZ(inactivityScoresBytes, 0); err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode inactivity scores: %w", err) } ret.SetInactivityScoresRaw(inactivityScores) - // Sync syncCommitteeSlot := r.cfg.RoundSlotToSyncCommitteePeriod(slot) currentSyncCommittee, err := state_accessors.ReadCurrentSyncCommittee(tx, syncCommitteeSlot) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read current sync committee: %w", err) } if currentSyncCommittee == nil { currentSyncCommittee = r.genesisState.CurrentSyncCommittee() @@ -224,7 +223,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. nextSyncCommittee, err := state_accessors.ReadNextSyncCommittee(tx, syncCommitteeSlot) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read next sync committee: %w", err) } if nextSyncCommittee == nil { nextSyncCommittee = r.genesisState.NextSyncCommittee() @@ -233,30 +232,30 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv. ret.SetNextSyncCommittee(nextSyncCommittee) // Execution if ret.Version() < clparams.BellatrixVersion { - return ret, nil + return ret, ret.InitBeaconState() } payloadHeader, err := block.Block.Body.ExecutionPayload.PayloadHeader() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read payload header: %w", err) } ret.SetLatestExecutionPayloadHeader(payloadHeader) if ret.Version() < clparams.CapellaVersion { - return ret, nil + return ret, ret.InitBeaconState() } // Withdrawals ret.SetNextWithdrawalIndex(minimalBeaconState.NextWithdrawalIndex) ret.SetNextWithdrawalValidatorIndex(minimalBeaconState.NextWithdrawalValidatorIndex) // Deep history valid from Capella onwards - historicalSummaries := solid.NewDynamicListSSZ[*cltypes.HistoricalSummary](int(r.cfg.HistoricalRootsLimit)) + historicalSummaries := solid.NewStaticListSSZ[*cltypes.HistoricalSummary](int(r.cfg.HistoricalRootsLimit), 64) if err := state_accessors.ReadHistoricalSummaries(tx, minimalBeaconState.HistoricalSummariesLength, func(idx int, historicalSummary *cltypes.HistoricalSummary) error { historicalSummaries.Append(historicalSummary) return nil }); err != nil { - return nil, err + return nil, fmt.Errorf("failed to read historical summaries: %w", err) } ret.SetHistoricalSummaries(historicalSummaries) - return ret, nil + return ret, ret.InitBeaconState() } func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector solid.HashVectorSSZ, slot, size uint64, table string, out solid.HashVectorSSZ) (err error) { @@ -290,7 +289,7 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s return nil } -func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, slot uint64, out *solid.ListSSZ[*cltypes.Eth1Data]) error { +func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength, slot uint64, out *solid.ListSSZ[*cltypes.Eth1Data]) error { initialSlot := r.cfg.RoundSlotToVotePeriod(slot) initialKey := base_encoding.Encode64ToBytes4(initialSlot) cursor, err := tx.Cursor(kv.Eth1DataVotes) @@ -302,17 +301,19 @@ func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, slot uint64, out *s if err != nil { return err } - for initialSlot > base_encoding.Decode64FromBytes4(k) { - k, v, err = cursor.Next() - if err != nil { - return err - } - if k == nil { - return fmt.Errorf("eth1 data votes not found for slot %d", slot) + if initialSlot <= r.genesisState.Slot() { + // We need to prepend the genesis votes + for i := 0; i < r.genesisState.Eth1DataVotes().Len(); i++ { + out.Append(r.genesisState.Eth1DataVotes().Get(i)) } } + endSlot := r.cfg.RoundSlotToVotePeriod(slot + r.cfg.SlotsPerEpoch*r.cfg.EpochsPerEth1VotingPeriod) + for k != nil && base_encoding.Decode64FromBytes4(k) < endSlot { + if out.Len() >= int(eth1DataVotesLength) { + break + } eth1Data := &cltypes.Eth1Data{} if err := eth1Data.DecodeSSZ(v, 0); err != nil { return err @@ -394,13 +395,10 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, slot uint } currentList := make([]byte, lenRaw) - if _, err := zstdReader.Read(currentList); err != nil { + if _, err = utils.ReadZSTD(zstdReader, currentList); err != nil { return nil, err } - if freshDumpSlot == slot { - return currentList, nil - } // now start diffing diffCursor, err := tx.Cursor(diffBucket) if err != nil { @@ -453,20 +451,16 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64Vector(tx kv.Tx, slot ui return nil, err } defer zstdReader.Close() - currentList := make([]byte, size*8) var n int - if n, err = zstdReader.Read(currentList); err != nil && !errors.Is(err, io.EOF) { - return nil, err + + if n, err = utils.ReadZSTD(zstdReader, currentList); err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("failed to read dump: %w, len: %d", err, len(v)) } if n != size*8 { return nil, err } - if freshDumpSlot == slot { - return out, out.DecodeSSZ(currentList, 0) - } - for k, v, err := diffCursor.Next(); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= slot; k, v, err = diffCursor.Next() { if len(k) != 4 { return nil, fmt.Errorf("invalid key %x", k) @@ -509,6 +503,8 @@ func (r *HistoricalStatesReader) readValidatorsForHistoricalState(tx kv.Tx, slot } return true }) + // Read the balances + bytesEffectiveBalances, err := r.reconstructDiffedUint64List(tx, slot, kv.ValidatorEffectiveBalance, "effective_balances") if err != nil { return nil, nil, nil, err @@ -550,6 +546,22 @@ func (r *HistoricalStatesReader) readPartecipations(tx kv.Tx, slot uint64, valid if err != nil { return nil, nil, err } + // trigger the cache for shuffled sets in parallel + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _, _ = r.computeCommittee(ret.RandaoMixes(), currentActiveIndicies, slot, r.cfg.TargetCommitteeSize, 0) + }() + if slot > r.cfg.SlotsPerEpoch { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = r.computeCommittee(ret.RandaoMixes(), previousActiveIndicies, slot-r.cfg.SlotsPerEpoch, r.cfg.TargetCommitteeSize, 0) + }() + } + wg.Wait() + // Read the previous idxs for i := beginSlot; i <= slot; i++ { // Read the block @@ -560,6 +572,7 @@ func (r *HistoricalStatesReader) readPartecipations(tx kv.Tx, slot uint64, valid if block == nil { continue } + ret.SetSlot(i) currentEpoch := slot / r.cfg.SlotsPerEpoch // Read the participation flags block.Block.Body.Attestations.Range(func(index int, attestation *solid.Attestation, length int) bool { @@ -630,7 +643,7 @@ func (r *HistoricalStatesReader) readPreviousPartecipation(slot, validatorLength return nil, err } - if _, err = zstdReader.Read(out); err != nil && !errors.Is(err, io.EOF) { + if _, err = utils.ReadZSTD(zstdReader, out); err != nil && !errors.Is(err, io.EOF) { return nil, err } ret := solid.NewBitList(0, int(r.cfg.ValidatorRegistryLimit)) diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go index 970b928fe15..13b51fbf1d5 100644 --- a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go +++ b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go @@ -28,13 +28,15 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt f := afero.NewMemMapFs() a := antiquary.NewAntiquary(ctx, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, nil, log.New(), true, f) require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33)) - // Now lets test it against the reader - hr := historical_states_reader.NewHistoricalStatesReader(&clparams.MainnetBeaconConfig, reader, vt, f, preState) tx, err := db.BeginRw(ctx) require.NoError(t, err) defer tx.Rollback() + vt = state_accessors.NewStaticValidatorTable() + require.NoError(t, state_accessors.ReadValidatorsTable(tx, vt)) + hr := historical_states_reader.NewHistoricalStatesReader(&clparams.MainnetBeaconConfig, reader, vt, f, preState) + s, err := hr.ReadHistoricalState(ctx, tx, blocks[len(blocks)-1].Block.Slot) require.NoError(t, err) @@ -43,17 +45,22 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt postHash2, err := postState.HashSSZ() require.NoError(t, err) require.Equal(t, libcommon.Hash(postHash2), libcommon.Hash(postHash)) - } func TestStateAntiquaryCapella(t *testing.T) { - t.Skip() + //t.Skip() blocks, preState, postState := tests.GetCapellaRandom() runTest(t, blocks, preState, postState) } func TestStateAntiquaryPhase0(t *testing.T) { - t.Skip() + // t.Skip() blocks, preState, postState := tests.GetPhase0Random() runTest(t, blocks, preState, postState) } + +func TestStateAntiquaryBellatrix(t *testing.T) { + // t.Skip() + blocks, preState, postState := tests.GetBellatrixRandom() + runTest(t, blocks, preState, postState) +} diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go index d5301323e8c..6155bde3371 100644 --- a/cl/persistence/state/state_accessors.go +++ b/cl/persistence/state/state_accessors.go @@ -265,3 +265,34 @@ func ReadPreviousEpochAttestations(tx kv.Tx, slot uint64, limit int) (*solid.Lis } return attestations, nil } + +func ReadValidatorsTable(tx kv.Tx, out *StaticValidatorTable) error { + cursor, err := tx.Cursor(kv.StaticValidators) + if err != nil { + return err + } + defer cursor.Close() + + var buf bytes.Buffer + for k, v, err := cursor.First(); err == nil && k != nil; k, v, err = cursor.Next() { + staticValidator := &StaticValidator{} + buf.Reset() + if _, err := buf.Write(v); err != nil { + return err + } + if err := staticValidator.ReadFrom(&buf); err != nil { + return err + } + out.validatorTable = append(out.validatorTable, staticValidator) + } + if err != nil { + return err + } + slot, err := GetStateProcessingProgress(tx) + if err != nil { + return err + } + out.slot = slot + return err + +} diff --git a/cl/persistence/state/static_validator_table.go b/cl/persistence/state/static_validator_table.go index f1f327ed054..5e32b890e2a 100644 --- a/cl/persistence/state/static_validator_table.go +++ b/cl/persistence/state/static_validator_table.go @@ -226,6 +226,9 @@ func NewStaticValidatorTable() *StaticValidatorTable { func (s *StaticValidatorTable) AddValidator(v solid.Validator, validatorIndex, slot uint64) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } s.validatorTable = append(s.validatorTable, NewStaticValidatorFromValidator(v, slot)) if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") @@ -236,6 +239,9 @@ func (s *StaticValidatorTable) AddValidator(v solid.Validator, validatorIndex, s func (s *StaticValidatorTable) AddWithdrawalCredentials(validatorIndex, slot uint64, withdrawalCredentials libcommon.Hash) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -246,6 +252,9 @@ func (s *StaticValidatorTable) AddWithdrawalCredentials(validatorIndex, slot uin func (s *StaticValidatorTable) AddSlashed(validatorIndex, slot uint64, slashed bool) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -256,6 +265,9 @@ func (s *StaticValidatorTable) AddSlashed(validatorIndex, slot uint64, slashed b func (s *StaticValidatorTable) AddActivationEligibility(validatorIndex, slot uint64, activationEligibility uint64) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -266,6 +278,9 @@ func (s *StaticValidatorTable) AddActivationEligibility(validatorIndex, slot uin func (s *StaticValidatorTable) AddActivationEpoch(validatorIndex, slot uint64, activationEpoch uint64) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -276,6 +291,9 @@ func (s *StaticValidatorTable) AddActivationEpoch(validatorIndex, slot uint64, a func (s *StaticValidatorTable) AddExitEpoch(validatorIndex, slot uint64, exitEpoch uint64) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -286,6 +304,9 @@ func (s *StaticValidatorTable) AddExitEpoch(validatorIndex, slot uint64, exitEpo func (s *StaticValidatorTable) AddWithdrawableEpoch(validatorIndex, slot uint64, withdrawableEpoch uint64) error { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return nil + } if validatorIndex >= uint64(len(s.validatorTable)) { return fmt.Errorf("validator index mismatch") } @@ -355,6 +376,9 @@ func (s *StaticValidatorTable) GetStaticValidator(validatorIndex uint64) *Static func (s *StaticValidatorTable) SetSlot(slot uint64) { s.sync.Lock() defer s.sync.Unlock() + if slot <= s.slot && s.slot != 0 { + return + } s.slot = slot } diff --git a/cl/persistence/state/validator_events_test.go b/cl/persistence/state/validator_events_test.go index 893c7a78640..b36be719f5a 100644 --- a/cl/persistence/state/validator_events_test.go +++ b/cl/persistence/state/validator_events_test.go @@ -1,7 +1,6 @@ package state_accessors import ( - "fmt" "testing" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -28,7 +27,6 @@ func TestStateEvents(t *testing.T) { events.ChangeSlashed(2, true) // Ok now lets replay it. ReplayEvents(func(validatorIndex uint64, validator solid.Validator) error { - fmt.Println(validator) require.Equal(t, validator, solid.NewValidator()) return nil }, func(validatorIndex, exitEpoch uint64) error { diff --git a/cl/phase1/core/state/cache.go b/cl/phase1/core/state/cache.go index b69956fe741..fcd410f080d 100644 --- a/cl/phase1/core/state/cache.go +++ b/cl/phase1/core/state/cache.go @@ -45,7 +45,7 @@ func New(cfg *clparams.BeaconChainConfig) *CachingBeaconState { state := &CachingBeaconState{ BeaconState: raw.New(cfg), } - state.initBeaconState() + state.InitBeaconState() return state } @@ -53,7 +53,7 @@ func NewFromRaw(r *raw.BeaconState) *CachingBeaconState { state := &CachingBeaconState{ BeaconState: r, } - state.initBeaconState() + state.InitBeaconState() return state } @@ -216,7 +216,7 @@ func (b *CachingBeaconState) initCaches() error { return nil } -func (b *CachingBeaconState) initBeaconState() error { +func (b *CachingBeaconState) InitBeaconState() error { b._refreshActiveBalances() b.publicKeyIndicies = make(map[[48]byte]uint64) diff --git a/cl/phase1/core/state/cache_accessors.go b/cl/phase1/core/state/cache_accessors.go index f777dacac5c..df6c7d53085 100644 --- a/cl/phase1/core/state/cache_accessors.go +++ b/cl/phase1/core/state/cache_accessors.go @@ -146,7 +146,16 @@ func (b *CachingBeaconState) GetAttestationParticipationFlagIndicies(data solid. } // Matching roots if !data.Source().Equal(justifiedCheckpoint) { - return nil, fmt.Errorf("GetAttestationParticipationFlagIndicies: source does not match") + // jsonify the data.Source and justifiedCheckpoint + jsonSource, err := data.Source().MarshalJSON() + if err != nil { + return nil, err + } + jsonJustifiedCheckpoint, err := justifiedCheckpoint.MarshalJSON() + if err != nil { + return nil, err + } + return nil, fmt.Errorf("GetAttestationParticipationFlagIndicies: source does not match. source: %s, justifiedCheckpoint: %s", jsonSource, jsonJustifiedCheckpoint) } targetRoot, err := GetBlockRoot(b, data.Target().Epoch()) if err != nil { diff --git a/cl/phase1/core/state/copy.go b/cl/phase1/core/state/copy.go index ebf0a111b53..de4ecd63538 100644 --- a/cl/phase1/core/state/copy.go +++ b/cl/phase1/core/state/copy.go @@ -22,7 +22,7 @@ func (b *CachingBeaconState) CopyInto(bs *CachingBeaconState) (err error) { func (b *CachingBeaconState) copyCachesInto(bs *CachingBeaconState) error { if b.Version() == clparams.Phase0Version { - return bs.initBeaconState() + return bs.InitBeaconState() } if bs.publicKeyIndicies == nil { bs.publicKeyIndicies = make(map[[48]byte]uint64) diff --git a/cl/phase1/core/state/raw/copy.go b/cl/phase1/core/state/raw/copy.go index f491ea1f5e4..86aa9e5e75a 100644 --- a/cl/phase1/core/state/raw/copy.go +++ b/cl/phase1/core/state/raw/copy.go @@ -16,7 +16,7 @@ func (b *BeaconState) CopyInto(dst *BeaconState) error { b.stateRoots.CopyTo(dst.stateRoots) b.historicalRoots.CopyTo(dst.historicalRoots) dst.eth1Data = b.eth1Data.Copy() - dst.eth1DataVotes = solid.NewDynamicListSSZ[*cltypes.Eth1Data](int(b.beaconConfig.Eth1DataVotesLength())) + dst.eth1DataVotes = solid.NewStaticListSSZ[*cltypes.Eth1Data](int(b.beaconConfig.Eth1DataVotesLength()), 72) b.eth1DataVotes.Range(func(index int, value *cltypes.Eth1Data, length int) bool { dst.eth1DataVotes.Append(value.Copy()) return true diff --git a/cl/phase1/core/state/ssz.go b/cl/phase1/core/state/ssz.go index 81b09c748e0..057666abfa9 100644 --- a/cl/phase1/core/state/ssz.go +++ b/cl/phase1/core/state/ssz.go @@ -25,7 +25,7 @@ func (b *CachingBeaconState) DecodeSSZ(buf []byte, version int) error { sz := metrics.NewHistTimer("decode_ssz_beacon_state_size") sz.Observe(float64(len(buf))) h.PutSince() - return b.initBeaconState() + return b.InitBeaconState() } // SSZ size of the Beacon State diff --git a/cl/rpc/rpc.go b/cl/rpc/rpc.go index c71501dbb1e..0ada88e8115 100644 --- a/cl/rpc/rpc.go +++ b/cl/rpc/rpc.go @@ -143,7 +143,7 @@ func (b *BeaconRpcP2P) SendBeaconBlocksByRangeReq(ctx context.Context, start, co // SendBeaconBlocksByRootReq retrieves blocks by root from beacon chain. func (b *BeaconRpcP2P) SendBeaconBlocksByRootReq(ctx context.Context, roots [][32]byte) ([]*cltypes.SignedBeaconBlock, string, error) { - var req solid.HashListSSZ = solid.NewHashList(69696969) + var req solid.HashListSSZ = solid.NewHashList(69696969) // The number is used for hashing, it is innofensive here. for _, root := range roots { req.Append(root) } diff --git a/cl/spectest/consensus_tests/finality.go b/cl/spectest/consensus_tests/finality.go index a313c8a72ab..9a1de019a58 100644 --- a/cl/spectest/consensus_tests/finality.go +++ b/cl/spectest/consensus_tests/finality.go @@ -2,11 +2,12 @@ package consensus_tests import ( "fmt" - "github.com/ledgerwatch/erigon/cl/transition/machine" - "github.com/ledgerwatch/erigon/spectest" "io/fs" "testing" + "github.com/ledgerwatch/erigon/cl/transition/machine" + "github.com/ledgerwatch/erigon/spectest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/cl/spectest/consensus_tests/ssz_static.go b/cl/spectest/consensus_tests/ssz_static.go index 939078efe19..a1512837d44 100644 --- a/cl/spectest/consensus_tests/ssz_static.go +++ b/cl/spectest/consensus_tests/ssz_static.go @@ -57,7 +57,6 @@ func getSSZStaticConsensusTest[T unmarshalerMarshalerHashable](ref T) spectest.H require.NoError(t, err) require.EqualValues(t, expectedRoot, haveRoot) // Cannot test it without a config. - // TODO: parse and use config if isBeaconState { return nil } diff --git a/cl/ssz/decode.go b/cl/ssz/decode.go index 1cd1b2e46c4..1308081adf7 100644 --- a/cl/ssz/decode.go +++ b/cl/ssz/decode.go @@ -42,7 +42,6 @@ func UnmarshalSSZ(buf []byte, version int, schema ...interface{}) (err error) { // err = fmt.Errorf("panic while decoding: %v", err2) // } // }() - position := 0 offsets := []int{} dynamicObjs := []SizedObjectSSZ{} diff --git a/cl/ssz/encode.go b/cl/ssz/encode.go index 2f851f1d208..38b57a8a11c 100644 --- a/cl/ssz/encode.go +++ b/cl/ssz/encode.go @@ -57,6 +57,7 @@ func MarshalSSZ(buf []byte, schema ...any) (dst []byte, err error) { err = fmt.Errorf("panic while encoding: %v", err2) } }() + dst = buf currentOffset := 0 dynamicComponents := []SizedObjectSSZ{} diff --git a/cl/transition/impl/eth2/operations.go b/cl/transition/impl/eth2/operations.go index 1a1ea5224ad..e872cbe45d1 100644 --- a/cl/transition/impl/eth2/operations.go +++ b/cl/transition/impl/eth2/operations.go @@ -365,7 +365,7 @@ func processSyncAggregate(s abstract.BeaconState, sync *cltypes.SyncAggregate) ( vIdx, exists := s.ValidatorIndexByPubkey(committeeKeys[currPubKeyIndex]) // Impossible scenario. if !exists { - return nil, errors.New("validator public key does not exist in state") + return nil, fmt.Errorf("validator public key does not exist in state: %x", committeeKeys[currPubKeyIndex]) } if syncAggregateBits[i]&byte(bit) > 0 { votedKeys = append(votedKeys, committeeKeys[currPubKeyIndex][:]) diff --git a/cl/utils/bytes.go b/cl/utils/bytes.go index 417fe4b583c..387bad6f5de 100644 --- a/cl/utils/bytes.go +++ b/cl/utils/bytes.go @@ -15,6 +15,7 @@ package utils import ( "encoding/binary" + "io" "math/bits" "github.com/ledgerwatch/erigon-lib/types/ssz" @@ -103,3 +104,17 @@ func GetBitlistLength(b []byte) int { // bit. Subtract this value by 1 to determine the length of the bitlist. return 8*(len(b)-1) + msb - 1 } + +func ReadZSTD(r io.Reader, out []byte) (int, error) { + n := 0 + var err error + for n != len(out) { + var m int + m, err = r.Read(out[n:]) + n += m + if err != nil { + return n, err + } + } + return n, nil +} diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index bfbd85bcb57..30ff9186fb1 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "os" "strings" "time" @@ -11,6 +12,7 @@ import ( "github.com/ledgerwatch/erigon/turbo/debug" lg "github.com/anacrolix/log" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/direct" downloader3 "github.com/ledgerwatch/erigon-lib/downloader" "github.com/ledgerwatch/erigon-lib/metrics" @@ -23,6 +25,7 @@ import ( "github.com/ledgerwatch/erigon/cl/abstract" "github.com/ledgerwatch/erigon/cl/antiquary" "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/clparams/initial_state" "github.com/ledgerwatch/erigon/cl/cltypes" persistence2 "github.com/ledgerwatch/erigon/cl/persistence" "github.com/ledgerwatch/erigon/cmd/caplin/caplin1" @@ -40,6 +43,9 @@ import ( "github.com/ledgerwatch/erigon/cl/persistence/beacon_indicies" "github.com/ledgerwatch/erigon/cl/persistence/db_config" "github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format" + "github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format/getters" + state_accessors "github.com/ledgerwatch/erigon/cl/persistence/state" + "github.com/ledgerwatch/erigon/cl/persistence/state/historical_states_reader" "github.com/ledgerwatch/erigon/cl/phase1/core" "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/phase1/network" @@ -64,11 +70,12 @@ var CLI struct { Blocks Blocks `cmd:"" help:"download blocks from reqresp network"` Epochs Epochs `cmd:"" help:"download epochs from reqresp network"` - Chain Chain `cmd:"" help:"download the entire chain from reqresp network"` - DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"` - CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"` - DownloadSnapshots DownloadSnapshots `cmd:"" help:"download snapshots from webseed"` - LoopSnapshots LoopSnapshots `cmd:"" help:"loop over snapshots"` + Chain Chain `cmd:"" help:"download the entire chain from reqresp network"` + DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"` + CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"` + DownloadSnapshots DownloadSnapshots `cmd:"" help:"download snapshots from webseed"` + LoopSnapshots LoopSnapshots `cmd:"" help:"loop over snapshots"` + RetrieveHistoricalState RetrieveHistoricalState `cmd:"" help:"retrieve historical state from db"` } type chainCfg struct { @@ -650,3 +657,97 @@ func (d *DownloadSnapshots) Run(ctx *Context) error { } return snapshotsync.WaitForDownloader("CapCliDownloader", ctx, false, snapshotsync.OnlyCaplin, s, tx, freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root()), freezeblocks.NewBorRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root())), nil, params.ChainConfigByChainName(d.Chain), direct.NewDownloaderClient(bittorrentServer)) } + +type RetrieveHistoricalState struct { + chainCfg + outputFolder + CompareFile string `help:"compare file" default:""` + CompareSlot uint64 `help:"compare slot" default:"0"` +} + +func (r *RetrieveHistoricalState) Run(ctx *Context) error { + vt := state_accessors.NewStaticValidatorTable() + _, _, beaconConfig, t, err := clparams.GetConfigsByNetworkName(r.Chain) + if err != nil { + return err + } + dirs := datadir.New(r.Datadir) + rawDB, fs := persistence.AferoRawBeaconBlockChainFromOsPath(beaconConfig, dirs.CaplinHistory) + beaconDB, db, err := caplin1.OpenCaplinDatabase(ctx, db_config.DatabaseConfiguration{PruneDepth: math.MaxUint64}, beaconConfig, rawDB, dirs.CaplinIndexing, nil, false) + if err != nil { + return err + } + log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler)) + + tx, err := db.BeginRo(ctx) + if err != nil { + return err + } + defer tx.Rollback() + allSnapshots := freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root()) + if err := allSnapshots.ReopenFolder(); err != nil { + return err + } + if err := state_accessors.ReadValidatorsTable(tx, vt); err != nil { + return err + } + fmt.Println(allSnapshots.BlocksAvailable(), allSnapshots.Dir()) + + var bor *freezeblocks.BorRoSnapshots + blockReader := freezeblocks.NewBlockReader(allSnapshots, bor) + eth1Getter := getters.NewExecutionSnapshotReader(ctx, blockReader, db) + csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root()) + if err := csn.ReopenFolder(); err != nil { + return err + } + snr := freezeblocks.NewBeaconSnapshotReader(csn, eth1Getter, beaconDB, beaconConfig) + gSpot, err := initial_state.GetGenesisState(t) + if err != nil { + return err + } + + hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, fs, gSpot) + start := time.Now() + haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot) + if err != nil { + return err + } + + v := haveState.Version() + // encode and decode the state + enc, err := haveState.EncodeSSZ(nil) + if err != nil { + return err + } + haveState = state.New(beaconConfig) + if err := haveState.DecodeSSZ(enc, int(v)); err != nil { + return err + } + endTime := time.Since(start) + hRoot, err := haveState.HashSSZ() + if err != nil { + return err + } + log.Info("Got state", "slot", haveState.Slot(), "root", libcommon.Hash(hRoot), "elapsed", endTime) + if r.CompareFile == "" { + return nil + } + // Read the content of CompareFile in a []byte without afero + rawBytes, err := os.ReadFile(r.CompareFile) + if err != nil { + return err + } + // Decode the []byte into a state + wantState := state.New(beaconConfig) + if err := wantState.DecodeSSZ(rawBytes, int(haveState.Version())); err != nil { + return err + } + wRoot, err := wantState.HashSSZ() + if err != nil { + return err + } + if hRoot != wRoot { + return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot)) + } + return nil +} diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go index be09dab6c74..0aba101d066 100644 --- a/cmd/caplin/caplin1/run.go +++ b/cmd/caplin/caplin1/run.go @@ -109,22 +109,6 @@ func RunCaplinPhase1(ctx context.Context, sentinel sentinel.SentinelClient, engi } } - vTables := state_accessors.NewStaticValidatorTable() - // get the initial state - genesisState, err := initial_state.GetGenesisState(clparams.NetworkType(beaconConfig.DepositNetworkID)) - if err != nil { - return err - } - antiq := antiquary.NewAntiquary(ctx, genesisState, vTables, beaconConfig, dirs, snDownloader, db, csn, rcsn, beaconDB, logger, states, af) - // Create the antiquary - if backfilling { - go func() { - if err := antiq.Loop(); err != nil { - logger.Error("Antiquary failed", "err", err) - } - }() - } - pool := pool.NewOperationsPool(beaconConfig) caplinFcuPath := path.Join(dirs.Tmp, "caplin-forkchoice") @@ -218,6 +202,26 @@ func RunCaplinPhase1(ctx context.Context, sentinel sentinel.SentinelClient, engi return err } + vTables := state_accessors.NewStaticValidatorTable() + // Read the the current table + if states { + if err := state_accessors.ReadValidatorsTable(tx, vTables); err != nil { + return err + } + } + // get the initial state + genesisState, err := initial_state.GetGenesisState(clparams.NetworkType(beaconConfig.DepositNetworkID)) + if err != nil { + return err + } + antiq := antiquary.NewAntiquary(ctx, genesisState, vTables, beaconConfig, dirs, snDownloader, db, csn, rcsn, beaconDB, logger, states, af) + // Create the antiquary + go func() { + if err := antiq.Loop(); err != nil { + logger.Error("Antiquary failed", "err", err) + } + }() + if err := tx.Commit(); err != nil { return err }