From 5567da9262dcb7156b789aa37e1bfd3d0bb6ffa9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Fri, 25 Oct 2024 23:08:08 +0200
Subject: [PATCH 001/131] save

---
 erigon-lib/downloader/snaptype/files.go       |   2 +
 .../freezeblocks/caplin_state_snapshots.go    | 531 ++++++++++++++++++
 2 files changed, 533 insertions(+)
 create mode 100644 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go

diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go
index 26cebf23e70..2fd09861fe9 100644
--- a/erigon-lib/downloader/snaptype/files.go
+++ b/erigon-lib/downloader/snaptype/files.go
@@ -156,6 +156,7 @@ func parseFileName(dir, fileName string) (res FileInfo, ok bool) {
 	if !ok {
 		return res, ok
 	}
+	res.TypeString = parts[3]
 	return res, ok
 }
 
@@ -243,6 +244,7 @@ type FileInfo struct {
 	From, To        uint64
 	name, Path, Ext string
 	Type            Type
+	TypeString      string // This is for giulio's generic snapshots
 }
 
 func (f FileInfo) TorrentFileExists() (bool, error) { return dir.FileExist(f.Path + ".torrent") }
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
new file mode 100644
index 00000000000..02be1550ef2
--- /dev/null
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -0,0 +1,531 @@
+// Copyright 2024 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
+
+package freezeblocks
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/tidwall/btree"
+
+	"github.com/erigontech/erigon-lib/log/v3"
+
+	"github.com/erigontech/erigon-lib/chain/snapcfg"
+	libcommon "github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/common/background"
+	"github.com/erigontech/erigon-lib/common/datadir"
+	"github.com/erigontech/erigon-lib/common/dbg"
+	"github.com/erigontech/erigon-lib/downloader/snaptype"
+	"github.com/erigontech/erigon-lib/kv"
+	"github.com/erigontech/erigon-lib/seg"
+
+	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/eth/ethconfig"
+)
+
+// value: chunked(ssz(SignedBeaconBlocks))
+// slot       -> beacon_slot_segment_offset
+
+type CaplinStateSnapshots struct {
+	indicesReady  atomic.Bool
+	segmentsReady atomic.Bool
+
+	Salt uint32
+
+	dirtySegmentsLock   sync.RWMutex
+	visibleSegmentsLock sync.RWMutex
+
+	// BeaconBlocks *segments
+	// BlobSidecars *segments
+	Segments      map[string]*segments
+	snapshotTypes SnapshotTypes
+
+	dir         string
+	tmpdir      string
+	segmentsMax atomic.Uint64 // all types of .seg files are available - up to this number
+	idxMax      atomic.Uint64 // all types of .idx files are available - up to this number
+	cfg         ethconfig.BlocksFreezing
+	logger      log.Logger
+	// allows for pruning segments - this is the min availible segment
+	segmentsMin atomic.Uint64
+	// chain cfg
+	beaconCfg *clparams.BeaconChainConfig
+}
+
+type KeyValueGetter func(numId uint64) ([]byte, []byte, error)
+
+type SnapshotTypes struct {
+	Types map[string]KeyValueGetter
+}
+
+// NewCaplinStateSnapshots - opens all snapshots. But to simplify everything:
+//   - it opens snapshots only on App start and immutable after
+//   - all snapshots of given blocks range must exist - to make this blocks range available
+//   - gaps are not allowed
+//   - segment have [from:to) semantic
+func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.BeaconChainConfig, dirs datadir.Dirs, snapshotTypes SnapshotTypes, logger log.Logger) *CaplinStateSnapshots {
+	// BeaconBlocks := &segments{
+	// 	DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
+	// }
+	// BlobSidecars := &segments{
+	// 	DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
+	// }
+	Segments := make(map[string]*segments)
+	for k := range snapshotTypes.Types {
+		Segments[k] = &segments{
+			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
+		}
+	}
+	c := &CaplinStateSnapshots{snapshotTypes: snapshotTypes, dir: dirs.Snap, tmpdir: dirs.Tmp, cfg: cfg, Segments: Segments, logger: logger, beaconCfg: beaconCfg}
+	c.recalcVisibleFiles()
+	return c
+}
+
+func (s *CaplinStateSnapshots) IndicesMax() uint64  { return s.idxMax.Load() }
+func (s *CaplinStateSnapshots) SegmentsMax() uint64 { return s.segmentsMax.Load() }
+
+func (s *CaplinStateSnapshots) LogStat(str string) {
+	s.logger.Info(fmt.Sprintf("[snapshots:%s] Stat", str),
+		"blocks", libcommon.PrettyCounter(s.SegmentsMax()+1), "indices", libcommon.PrettyCounter(s.IndicesMax()+1))
+}
+
+func (s *CaplinStateSnapshots) LS() {
+	if s == nil {
+		return
+	}
+	view := s.View()
+	defer view.Close()
+
+	for _, roTx := range view.roTxs {
+		if roTx != nil {
+			for _, seg := range roTx.VisibleSegments {
+				s.logger.Info("[agg] ", "f", seg.src.Decompressor.FileName(), "words", seg.src.Decompressor.Count())
+			}
+		}
+	}
+}
+
+func (s *CaplinStateSnapshots) SegFileNames(from, to uint64) []string {
+	view := s.View()
+	defer view.Close()
+
+	var res []string
+
+	for _, roTx := range view.roTxs {
+		if roTx == nil {
+			continue
+		}
+		for _, seg := range roTx.VisibleSegments {
+			if seg.from >= from && seg.to <= to {
+				res = append(res, seg.src.FileName())
+			}
+		}
+	}
+	return res
+}
+
+func (s *CaplinStateSnapshots) BlocksAvailable() uint64 {
+	return min(s.segmentsMax.Load(), s.idxMax.Load())
+}
+
+func (s *CaplinStateSnapshots) Close() {
+	if s == nil {
+		return
+	}
+	s.dirtySegmentsLock.Lock()
+	defer s.dirtySegmentsLock.Unlock()
+
+	s.closeWhatNotInList(nil)
+}
+
+// OpenList stops on optimistic=false, continue opening files on optimistic=true
+func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) error {
+	defer s.recalcVisibleFiles()
+
+	s.dirtySegmentsLock.Lock()
+	defer s.dirtySegmentsLock.Unlock()
+
+	s.closeWhatNotInList(fileNames)
+	var segmentsMax uint64
+	var segmentsMaxSet bool
+Loop:
+	for _, fName := range fileNames {
+		f, _, ok := snaptype.ParseFileName(s.dir, fName)
+		if !ok {
+			continue
+		}
+
+		var processed bool = true
+		var exists bool
+		var sn *DirtySegment
+		segments, ok := s.Segments[f.TypeString]
+		if !ok {
+			continue
+		}
+		segments.DirtySegments.Walk(func(segments []*DirtySegment) bool {
+			for _, sn2 := range segments {
+				if sn2.Decompressor == nil { // it's ok if some segment was not able to open
+					continue
+				}
+				if fName == sn2.FileName() {
+					sn = sn2
+					exists = true
+					break
+				}
+			}
+			return true
+		})
+		if !exists {
+			sn = &DirtySegment{
+				// segType: f.Type, Unsupported
+				version: f.Version,
+				Range:   Range{f.From, f.To},
+				frozen:  snapcfg.IsFrozen(s.cfg.ChainName, f),
+			}
+		}
+		if err := sn.openSegIfNeed(s.dir); err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				if optimistic {
+					continue Loop
+				} else {
+					break Loop
+				}
+			}
+			if optimistic {
+				s.logger.Warn("[snapshots] open segment", "err", err)
+				continue Loop
+			} else {
+				return err
+			}
+		}
+
+		if !exists {
+			// it's possible to iterate over .seg file even if you don't have index
+			// then make segment available even if index open may fail
+			segments.DirtySegments.Set(sn)
+		}
+		if err := sn.openIdxIfNeed(s.dir, optimistic); err != nil {
+			return err
+		}
+		// Only bob sidecars count for progression
+		if processed {
+			if f.To > 0 {
+				segmentsMax = f.To - 1
+			} else {
+				segmentsMax = 0
+			}
+			segmentsMaxSet = true
+		}
+	}
+
+	if segmentsMaxSet {
+		s.segmentsMax.Store(segmentsMax)
+	}
+	s.segmentsReady.Store(true)
+	return nil
+}
+
+func (s *CaplinStateSnapshots) recalcVisibleFiles() {
+	defer func() {
+		s.idxMax.Store(s.idxAvailability())
+		s.indicesReady.Store(true)
+	}()
+
+	s.visibleSegmentsLock.Lock()
+	defer s.visibleSegmentsLock.Unlock()
+
+	getNewVisibleSegments := func(dirtySegments *btree.BTreeG[*DirtySegment]) []*VisibleSegment {
+		newVisibleSegments := make([]*VisibleSegment, 0, dirtySegments.Len())
+		dirtySegments.Walk(func(segments []*DirtySegment) bool {
+			for _, sn := range segments {
+				if sn.canDelete.Load() {
+					continue
+				}
+				if !sn.Indexed() {
+					continue
+				}
+				for len(newVisibleSegments) > 0 && newVisibleSegments[len(newVisibleSegments)-1].src.isSubSetOf(sn) {
+					newVisibleSegments[len(newVisibleSegments)-1].src = nil
+					newVisibleSegments = newVisibleSegments[:len(newVisibleSegments)-1]
+				}
+				newVisibleSegments = append(newVisibleSegments, &VisibleSegment{
+					Range:   sn.Range,
+					segType: sn.segType,
+					src:     sn,
+				})
+			}
+			return true
+		})
+		return newVisibleSegments
+	}
+
+	for _, segments := range s.Segments {
+		segments.VisibleSegments = getNewVisibleSegments(segments.DirtySegments)
+		var maxIdx uint64
+		if len(segments.VisibleSegments) > 0 {
+			maxIdx = segments.VisibleSegments[len(segments.VisibleSegments)-1].to - 1
+		}
+		segments.maxVisibleBlock.Store(maxIdx)
+	}
+}
+
+func (s *CaplinStateSnapshots) idxAvailability() uint64 {
+	minVisible := uint64(math.MaxUint64)
+	for _, segments := range s.Segments {
+		if segments.maxVisibleBlock.Load() < minVisible {
+			minVisible = segments.maxVisibleBlock.Load()
+		}
+	}
+	if minVisible == math.MaxUint64 {
+		return 0
+	}
+	return minVisible
+}
+
+func (s *CaplinStateSnapshots) OpenFolder() error {
+	files, _, err := SegmentsCaplin(s.dir, s.segmentsMin.Load())
+	if err != nil {
+		return err
+	}
+	list := make([]string, 0, len(files))
+	for _, f := range files {
+		_, fName := filepath.Split(f.Path)
+		list = append(list, fName)
+	}
+	return s.OpenList(list, false)
+}
+
+func (s *CaplinStateSnapshots) closeWhatNotInList(l []string) {
+	protectFiles := make(map[string]struct{}, len(l))
+	for _, fName := range l {
+		protectFiles[fName] = struct{}{}
+	}
+
+	for _, segments := range s.Segments {
+		toClose := make([]*DirtySegment, 0)
+		segments.DirtySegments.Walk(func(segments []*DirtySegment) bool {
+			for _, sn := range segments {
+				if sn.Decompressor == nil {
+					continue
+				}
+				_, name := filepath.Split(sn.FilePath())
+				if _, ok := protectFiles[name]; ok {
+					continue
+				}
+				toClose = append(toClose, sn)
+			}
+			return true
+		})
+		for _, sn := range toClose {
+			sn.close()
+			segments.DirtySegments.Delete(sn)
+		}
+	}
+}
+
+type CaplinStateView struct {
+	s      *CaplinStateSnapshots
+	roTxs  map[string]*segmentsRotx
+	closed bool
+}
+
+func (s *CaplinStateSnapshots) View() *CaplinStateView {
+	s.visibleSegmentsLock.RLock()
+	defer s.visibleSegmentsLock.RUnlock()
+
+	v := &CaplinStateView{s: s}
+	// BeginRo increments refcount - which is contended
+	s.dirtySegmentsLock.RLock()
+	defer s.dirtySegmentsLock.RUnlock()
+
+	for k, segments := range s.Segments {
+		v.roTxs[k] = segments.BeginRotx()
+	}
+	return v
+}
+
+func (v *CaplinStateView) Close() {
+	if v.closed {
+		return
+	}
+	for _, segments := range v.roTxs {
+		segments.Close()
+	}
+	v.s = nil
+	v.closed = true
+}
+
+func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
+	return v.s.Segments[tbl].VisibleSegments
+}
+
+func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {
+	for _, seg := range v.VisibleSegments(tbl) {
+		if !(slot >= seg.from && slot < seg.to) {
+			continue
+		}
+		return seg, true
+	}
+	return nil, false
+}
+
+func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+	tmpDir, snapDir := dirs.Tmp, dirs.Snap
+
+	segName := snaptype.BeaconBlocks.FileName(0, fromSlot, toSlot)
+	// a little bit ugly.
+	segName = strings.ReplaceAll(segName, "beaconblocks", strings.ToLower(snapName))
+	f, _, _ := snaptype.ParseFileName(snapDir, segName)
+
+	compressCfg := seg.DefaultCfg
+	compressCfg.Workers = workers
+	sn, err := seg.NewCompressor(ctx, fmt.Sprintf("Snapshots %s", snapName), f.Path, tmpDir, compressCfg, lvl, logger)
+	if err != nil {
+		return err
+	}
+	defer sn.Close()
+
+	// Generate .seg file, which is just the list of beacon blocks.
+	for i := fromSlot; i < toSlot; i++ {
+		// read root.
+		_, dump, err := kvGetter(i)
+		if err != nil {
+			return err
+		}
+		if i%20_000 == 0 {
+			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
+		}
+		if err := sn.AddWord(dump); err != nil {
+			return err
+		}
+	}
+	if sn.Count() != snaptype.CaplinMergeLimit {
+		return fmt.Errorf("expected %d blocks, got %d", snaptype.CaplinMergeLimit, sn.Count())
+	}
+	if err := sn.Compress(); err != nil {
+		return fmt.Errorf("compress: %w", err)
+	}
+	// Generate .idx file, which is the slot => offset mapping.
+	p := &background.Progress{}
+
+	// Ugly hack to wait for fsync
+	time.Sleep(15 * time.Second)
+
+	return BeaconSimpleIdx(ctx, f, salt, tmpDir, p, lvl, logger)
+}
+
+func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+	cfg := snapcfg.KnownCfg("")
+	for snapName, kvGetter := range s.snapshotTypes.Types {
+		for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) {
+			blocksPerFile := snapcfg.MergeLimitFromCfg(cfg, snaptype.CaplinEnums.BeaconBlocks, i)
+
+			if toSlot-i < blocksPerFile {
+				break
+			}
+			// keep beaconblocks here but whatever....
+			to := chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil)
+			logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to)
+			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, salt, dirs, workers, lvl, logger); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (s *CaplinStateSnapshots) BuildMissingIndices(ctx context.Context, logger log.Logger) error {
+	if s == nil {
+		return nil
+	}
+	// if !s.segmentsReady.Load() {
+	// 	return fmt.Errorf("not all snapshot segments are available")
+	// }
+
+	// wait for Downloader service to download all expected snapshots
+	segments, _, err := SegmentsCaplin(s.dir, 0)
+	if err != nil {
+		return err
+	}
+	noneDone := true
+	for index := range segments {
+		segment := segments[index]
+		// The same slot=>offset mapping is used for both beacon blocks and blob sidecars.
+		if segment.Type.Enum() != snaptype.CaplinEnums.BeaconBlocks && segment.Type.Enum() != snaptype.CaplinEnums.BlobSidecars {
+			continue
+		}
+		if segment.Type.HasIndexFiles(segment, logger) {
+			continue
+		}
+		p := &background.Progress{}
+		noneDone = false
+		if err := BeaconSimpleIdx(ctx, segment, s.Salt, s.tmpdir, p, log.LvlDebug, logger); err != nil {
+			return err
+		}
+	}
+	if noneDone {
+		return nil
+	}
+
+	return s.OpenFolder()
+}
+
+func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			panic(fmt.Sprintf("ReadHeader(%d), %s, %s\n", slot, rec, dbg.Stack()))
+		}
+	}()
+
+	view := s.View()
+	defer view.Close()
+
+	var buf []byte
+
+	seg, ok := view.VisibleSegment(slot, tbl)
+	if !ok {
+		return nil, nil
+	}
+
+	idxSlot := seg.src.Index()
+
+	if idxSlot == nil {
+		return nil, nil
+	}
+	blockOffset := idxSlot.OrdinalLookup(slot - idxSlot.BaseDataID())
+
+	gg := seg.src.MakeGetter()
+	gg.Reset(blockOffset)
+	if !gg.HasNext() {
+		return nil, nil
+	}
+
+	buf, _ = gg.Next(buf)
+	if len(buf) == 0 {
+		return nil, nil
+	}
+
+	return buf, nil
+}

From 11dac56724c5dca8362aa518f63828b973f0d764 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Fri, 25 Oct 2024 23:24:13 +0200
Subject: [PATCH 002/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 02be1550ef2..9fbce70903d 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -94,7 +94,7 @@ func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.B
 	// }
 	Segments := make(map[string]*segments)
 	for k := range snapshotTypes.Types {
-		Segments[k] = &segments{
+		Segments[strings.ToLower(k)] = &segments{
 			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 		}
 	}

From af374401d32a9f0ecfd56636750eaf04eec0372d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Fri, 25 Oct 2024 23:54:17 +0200
Subject: [PATCH 003/131] save

---
 cmd/capcli/cli.go                             | 47 +++++++++++++++++++
 .../freezeblocks/caplin_state_snapshots.go    | 41 +++++++++++++++-
 2 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 86e1e809841..4141b635a7b 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -80,6 +81,7 @@ var CLI struct {
 	CheckBlobsSnapshots       CheckBlobsSnapshots       `cmd:"" help:"check blobs snapshots"`
 	CheckBlobsSnapshotsCount  CheckBlobsSnapshotsCount  `cmd:"" help:"check blobs snapshots count"`
 	DumpBlobsSnapshotsToStore DumpBlobsSnapshotsToStore `cmd:"" help:"dump blobs snapshots to store"`
+	DumpStateSnapshots        DumpStateSnapshots        `cmd:"" help:"dump state snapshots"`
 }
 
 type chainCfg struct {
@@ -1172,3 +1174,48 @@ func (c *DumpBlobsSnapshotsToStore) Run(ctx *Context) error {
 
 	return nil
 }
+
+type DumpStateSnapshots struct {
+	chainCfg
+	outputFolder
+	To uint64 `name:"to" help:"slot to dump"`
+}
+
+func (c *DumpStateSnapshots) Run(ctx *Context) error {
+	_, beaconConfig, _, err := clparams.GetConfigsByNetworkName(c.Chain)
+	if err != nil {
+		return err
+	}
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler))
+	log.Info("Started chain download", "chain", c.Chain)
+
+	dirs := datadir.New(c.Datadir)
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StderrHandler))
+
+	db, _, err := caplin1.OpenCaplinDatabase(ctx, beaconConfig, nil, dirs.CaplinIndexing, dirs.CaplinBlobs, nil, false, 0)
+	if err != nil {
+		return err
+	}
+	var to uint64
+	db.View(ctx, func(tx kv.Tx) (err error) {
+		if c.To == 0 {
+			to, err = state_accessors.GetStateProcessingProgress(tx)
+			return
+		}
+		to = c.To
+		return
+	})
+
+	salt, err := snaptype.GetIndexSalt(dirs.Snap)
+
+	if err != nil {
+		return err
+	}
+	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
+	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
+	stateSn.OpenFolder()
+	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 9fbce70903d..c3e7e2a9e34 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -42,9 +42,48 @@ import (
 	"github.com/erigontech/erigon-lib/seg"
 
 	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/cl/persistence/base_encoding"
 	"github.com/erigontech/erigon/eth/ethconfig"
 )
 
+func getKvGetterForStateTable(db kv.RoDB, tableName string) KeyValueGetter {
+	return func(numId uint64) ([]byte, []byte, error) {
+		var key, value []byte
+		var err error
+		if err := db.View(context.TODO(), func(tx kv.Tx) error {
+			key = base_encoding.Encode64ToBytes4(numId)
+			value, err = tx.GetOne(tableName, base_encoding.Encode64ToBytes4(numId))
+			return err
+		}); err != nil {
+			return nil, nil, err
+		}
+		return key, value, nil
+	}
+}
+
+func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
+	return SnapshotTypes{
+		Types: map[string]KeyValueGetter{
+			kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
+			kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
+			kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
+			kv.StateEvents:               getKvGetterForStateTable(db, kv.StateEvents),
+			kv.ActiveValidatorIndicies:   getKvGetterForStateTable(db, kv.ActiveValidatorIndicies),
+			kv.StateRoot:                 getKvGetterForStateTable(db, kv.StateRoot),
+			kv.BlockRoot:                 getKvGetterForStateTable(db, kv.BlockRoot),
+			kv.SlotData:                  getKvGetterForStateTable(db, kv.SlotData),
+			kv.EpochData:                 getKvGetterForStateTable(db, kv.EpochData),
+			kv.InactivityScores:          getKvGetterForStateTable(db, kv.InactivityScores),
+			kv.NextSyncCommittee:         getKvGetterForStateTable(db, kv.NextSyncCommittee),
+			kv.CurrentSyncCommittee:      getKvGetterForStateTable(db, kv.CurrentSyncCommittee),
+			kv.Eth1DataVotes:             getKvGetterForStateTable(db, kv.Eth1DataVotes),
+			kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
+			kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
+			kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
+		},
+	}
+}
+
 // value: chunked(ssz(SignedBeaconBlocks))
 // slot       -> beacon_slot_segment_offset
 
@@ -436,7 +475,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 	return BeaconSimpleIdx(ctx, f, salt, tmpDir, p, lvl, logger)
 }
 
-func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
 	cfg := snapcfg.KnownCfg("")
 	for snapName, kvGetter := range s.snapshotTypes.Types {
 		for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) {

From ae5f2baafbf895265a95dd4f4240813cf98fe477 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 01:55:20 +0200
Subject: [PATCH 004/131] save

---
 cl/antiquary/state_antiquary.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 7344fb52a89..efb5238954b 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -129,7 +129,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	defer tx.Rollback()
 
 	// maps which validators changes
-	changedValidators := make(map[uint64]struct{})
+	var changedValidators sync.Map
 
 	stateAntiquaryCollector := newBeaconStatesCollector(s.cfg, s.dirs.Tmp, s.logger)
 	defer stateAntiquaryCollector.close()
@@ -144,7 +144,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		}
 		// Mark all validators as touched because we just initizialized the whole state.
 		s.currentState.ForEachValidator(func(v solid.Validator, index, total int) bool {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			if err = s.validatorsTable.AddValidator(v, uint64(index), 0); err != nil {
 				return false
 			}
@@ -175,37 +175,37 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 			return stateAntiquaryCollector.collectIntraEpochRandaoMix(slot, mix)
 		},
 		OnNewValidator: func(index int, v solid.Validator, balance uint64) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.AddValidator(uint64(index), v)
 			return s.validatorsTable.AddValidator(v, uint64(index), slot)
 		},
 		OnNewValidatorActivationEpoch: func(index int, epoch uint64) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeActivationEpoch(uint64(index), epoch)
 			return s.validatorsTable.AddActivationEpoch(uint64(index), slot, epoch)
 		},
 		OnNewValidatorExitEpoch: func(index int, epoch uint64) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeExitEpoch(uint64(index), epoch)
 			return s.validatorsTable.AddExitEpoch(uint64(index), slot, epoch)
 		},
 		OnNewValidatorWithdrawableEpoch: func(index int, epoch uint64) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeWithdrawableEpoch(uint64(index), epoch)
 			return s.validatorsTable.AddWithdrawableEpoch(uint64(index), slot, epoch)
 		},
 		OnNewValidatorSlashed: func(index int, newSlashed bool) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeSlashed(uint64(index), newSlashed)
 			return s.validatorsTable.AddSlashed(uint64(index), slot, newSlashed)
 		},
 		OnNewValidatorActivationEligibilityEpoch: func(index int, epoch uint64) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeActivationEligibilityEpoch(uint64(index), epoch)
 			return s.validatorsTable.AddActivationEligibility(uint64(index), slot, epoch)
 		},
 		OnNewValidatorWithdrawalCredentials: func(index int, wc []byte) error {
-			changedValidators[uint64(index)] = struct{}{}
+			changedValidators.Store(uint64(index), struct{}{})
 			events.ChangeWithdrawalCredentials(uint64(index), libcommon.BytesToHash(wc))
 			return s.validatorsTable.AddWithdrawalCredentials(uint64(index), slot, libcommon.BytesToHash(wc))
 		},
@@ -389,7 +389,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 
 	buf := &bytes.Buffer{}
 	s.validatorsTable.ForEach(func(validatorIndex uint64, validator *state_accessors.StaticValidator) bool {
-		if _, ok := changedValidators[validatorIndex]; !ok {
+		if _, ok := changedValidators.Load(validatorIndex); !ok {
 			return true
 		}
 		buf.Reset()

From 4f8eca42e209e24bde95d6f79eae4a98514f172f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:06:50 +0200
Subject: [PATCH 005/131] save

---
 cl/persistence/state/validator_events.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/cl/persistence/state/validator_events.go b/cl/persistence/state/validator_events.go
index 3c5747c8166..e6a1c9e7e2a 100644
--- a/cl/persistence/state/validator_events.go
+++ b/cl/persistence/state/validator_events.go
@@ -19,6 +19,7 @@ package state_accessors
 import (
 	"encoding/binary"
 	"errors"
+	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon/cl/cltypes/solid"
@@ -42,6 +43,7 @@ const (
 
 type StateEvents struct {
 	buf []byte
+	mu  sync.Mutex
 }
 
 func NewStateEvents() *StateEvents {
@@ -49,42 +51,56 @@ func NewStateEvents() *StateEvents {
 }
 
 func (se *StateEvents) AddValidator(validatorIndex uint64, validator solid.Validator) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(addValidator))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = append(se.buf, validator...)
 }
 
 func (se *StateEvents) ChangeExitEpoch(validatorIndex uint64, exitEpoch uint64) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeExitEpoch))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = binary.BigEndian.AppendUint64(se.buf, exitEpoch)
 }
 
 func (se *StateEvents) ChangeWithdrawableEpoch(validatorIndex uint64, withdrawableEpoch uint64) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeWithdrawableEpoch))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = binary.BigEndian.AppendUint64(se.buf, withdrawableEpoch)
 }
 
 func (se *StateEvents) ChangeWithdrawalCredentials(validatorIndex uint64, withdrawalCredentials libcommon.Hash) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeWithdrawalCredentials))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = append(se.buf, withdrawalCredentials[:]...)
 }
 
 func (se *StateEvents) ChangeActivationEpoch(validatorIndex uint64, activationEpoch uint64) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeActivationEpoch))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = binary.BigEndian.AppendUint64(se.buf, activationEpoch)
 }
 
 func (se *StateEvents) ChangeActivationEligibilityEpoch(validatorIndex uint64, activationEligibilityEpoch uint64) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeActivationEligibilityEpoch))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = binary.BigEndian.AppendUint64(se.buf, activationEligibilityEpoch)
 }
 
 func (se *StateEvents) ChangeSlashed(validatorIndex uint64, slashed bool) {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = append(se.buf, byte(changeSlashed))
 	se.buf = binary.BigEndian.AppendUint64(se.buf, validatorIndex)
 	se.buf = append(se.buf, byte(0))
@@ -94,10 +110,14 @@ func (se *StateEvents) ChangeSlashed(validatorIndex uint64, slashed bool) {
 }
 
 func (se *StateEvents) CopyBytes() []byte {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	return libcommon.Copy(se.buf)
 }
 
 func (se *StateEvents) Reset() {
+	se.mu.Lock()
+	defer se.mu.Unlock()
 	se.buf = se.buf[:0]
 }
 

From 352332c06ed9f50fba7a802e8de7704fab459f92 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:36:28 +0200
Subject: [PATCH 006/131] save

---
 erigon-lib/common/datadir/dirs.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/erigon-lib/common/datadir/dirs.go b/erigon-lib/common/datadir/dirs.go
index 266625088f2..3a654eead38 100644
--- a/erigon-lib/common/datadir/dirs.go
+++ b/erigon-lib/common/datadir/dirs.go
@@ -42,6 +42,7 @@ type Dirs struct {
 	SnapHistory     string
 	SnapDomain      string
 	SnapAccessors   string
+	SnapCaplin      string
 	Downloader      string
 	TxPool          string
 	Nodes           string
@@ -79,6 +80,7 @@ func New(datadir string) Dirs {
 		CaplinIndexing:  filepath.Join(datadir, "caplin", "indexing"),
 		CaplinLatest:    filepath.Join(datadir, "caplin", "latest"),
 		CaplinGenesis:   filepath.Join(datadir, "caplin", "genesis"),
+		SnapCaplin:      filepath.Join(datadir, "snapshots", "caplin"),
 	}
 
 	dir.MustExist(dirs.Chaindata, dirs.Tmp,

From 9d9bd2fb5e4924daaf56ba5509cec2293203deff Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:37:27 +0200
Subject: [PATCH 007/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index c3e7e2a9e34..e2ce767ba82 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -137,7 +137,7 @@ func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.B
 			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 		}
 	}
-	c := &CaplinStateSnapshots{snapshotTypes: snapshotTypes, dir: dirs.Snap, tmpdir: dirs.Tmp, cfg: cfg, Segments: Segments, logger: logger, beaconCfg: beaconCfg}
+	c := &CaplinStateSnapshots{snapshotTypes: snapshotTypes, dir: dirs.SnapCaplin, tmpdir: dirs.Tmp, cfg: cfg, Segments: Segments, logger: logger, beaconCfg: beaconCfg}
 	c.recalcVisibleFiles()
 	return c
 }
@@ -431,7 +431,7 @@ func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegme
 }
 
 func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
-	tmpDir, snapDir := dirs.Tmp, dirs.Snap
+	tmpDir, snapDir := dirs.Tmp, dirs.SnapCaplin
 
 	segName := snaptype.BeaconBlocks.FileName(0, fromSlot, toSlot)
 	// a little bit ugly.

From c62a248bcccea99e3a73c3fc818af1b5b806f906 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:40:20 +0200
Subject: [PATCH 008/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index e2ce767ba82..3bd26994576 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -463,9 +463,6 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 	if sn.Count() != snaptype.CaplinMergeLimit {
 		return fmt.Errorf("expected %d blocks, got %d", snaptype.CaplinMergeLimit, sn.Count())
 	}
-	if err := sn.Compress(); err != nil {
-		return fmt.Errorf("compress: %w", err)
-	}
 	// Generate .idx file, which is the slot => offset mapping.
 	p := &background.Progress{}
 

From bda7346e5486570e2a6b2718adffce719fccefb9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:42:14 +0200
Subject: [PATCH 009/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 3bd26994576..e2ce767ba82 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -463,6 +463,9 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 	if sn.Count() != snaptype.CaplinMergeLimit {
 		return fmt.Errorf("expected %d blocks, got %d", snaptype.CaplinMergeLimit, sn.Count())
 	}
+	if err := sn.Compress(); err != nil {
+		return fmt.Errorf("compress: %w", err)
+	}
 	// Generate .idx file, which is the slot => offset mapping.
 	p := &background.Progress{}
 

From 9a01dff8a9c3e03b2193f66e5a59ecba5438e35c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:51:28 +0200
Subject: [PATCH 010/131] save

---
 erigon-lib/downloader/snaptype/type.go        | 61 +++++++++++++++++++
 .../freezeblocks/caplin_state_snapshots.go    | 31 +++++++++-
 2 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/snaptype/type.go b/erigon-lib/downloader/snaptype/type.go
index 966a67ddf78..cfaff822251 100644
--- a/erigon-lib/downloader/snaptype/type.go
+++ b/erigon-lib/downloader/snaptype/type.go
@@ -469,6 +469,67 @@ func BuildIndex(ctx context.Context, info FileInfo, cfg recsplit.RecSplitArgs, l
 	}
 }
 
+func BuildIndexWithSnapName(ctx context.Context, info FileInfo, cfg recsplit.RecSplitArgs, lvl log.Lvl, p *background.Progress, walker func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error, logger log.Logger) (err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = fmt.Errorf("index panic: at=%s, %v, %s", info.Name(), rec, dbg.Stack())
+		}
+	}()
+
+	d, err := seg.NewDecompressor(info.Path)
+	if err != nil {
+		return fmt.Errorf("can't open %s for indexing: %w", info.Name(), err)
+	}
+	defer d.Close()
+
+	if p != nil {
+		fname := info.Name()
+		p.Name.Store(&fname)
+		p.Total.Store(uint64(d.Count()))
+	}
+	cfg.KeyCount = d.Count()
+	cfg.IndexFile = filepath.Join(info.Dir(), strings.ReplaceAll(info.name, ".seg", ".idx"))
+	rs, err := recsplit.NewRecSplit(cfg, logger)
+	if err != nil {
+		return err
+	}
+	rs.LogLvl(lvl)
+
+	defer d.EnableReadAhead().DisableReadAhead()
+
+	for {
+		g := d.MakeGetter()
+		var i, offset, nextPos uint64
+		word := make([]byte, 0, 4096)
+
+		for g.HasNext() {
+			word, nextPos = g.Next(word[:0])
+			if err := walker(rs, i, offset, word); err != nil {
+				return err
+			}
+			i++
+			offset = nextPos
+
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			default:
+			}
+		}
+
+		if err = rs.Build(ctx); err != nil {
+			if errors.Is(err, recsplit.ErrCollision) {
+				logger.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err)
+				rs.ResetNextSalt()
+				continue
+			}
+			return err
+		}
+
+		return nil
+	}
+}
+
 func ExtractRange(ctx context.Context, f FileInfo, extractor RangeExtractor, firstKey FirstKeyGetter, chainDB kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
 	var lastKeyValue uint64
 
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index e2ce767ba82..0356d7932c6 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -18,6 +18,7 @@ package freezeblocks
 
 import (
 	"context"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"math"
@@ -31,6 +32,7 @@ import (
 	"github.com/tidwall/btree"
 
 	"github.com/erigontech/erigon-lib/log/v3"
+	"github.com/erigontech/erigon-lib/recsplit"
 
 	"github.com/erigontech/erigon-lib/chain/snapcfg"
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -472,7 +474,34 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 	// Ugly hack to wait for fsync
 	time.Sleep(15 * time.Second)
 
-	return BeaconSimpleIdx(ctx, f, salt, tmpDir, p, lvl, logger)
+	return simpleIdx(ctx, f, salt, tmpDir, p, lvl, logger)
+}
+
+func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
+	num := make([]byte, binary.MaxVarintLen64)
+	cfg := recsplit.RecSplitArgs{
+		Enums:      true,
+		BucketSize: 2000,
+		LeafSize:   8,
+		TmpDir:     tmpDir,
+		Salt:       &salt,
+		BaseDataID: sn.From,
+	}
+	if err := snaptype.BuildIndexWithSnapName(ctx, sn, cfg, log.LvlDebug, p, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
+		if i%20_000 == 0 {
+			logger.Log(lvl, "Generating idx for "+sn.Type.Name(), "progress", i)
+		}
+		p.Processed.Add(1)
+		n := binary.PutUvarint(num, i)
+		if err := idx.AddKey(num[:n], offset); err != nil {
+			return err
+		}
+		return nil
+	}, logger); err != nil {
+		return fmt.Errorf("idx: %w", err)
+	}
+
+	return nil
 }
 
 func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {

From 93fc7499dfa4802c00c8af6734213136e9f83463 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:52:45 +0200
Subject: [PATCH 011/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 0356d7932c6..dc9fcc06d91 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -489,7 +489,7 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st
 	}
 	if err := snaptype.BuildIndexWithSnapName(ctx, sn, cfg, log.LvlDebug, p, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
 		if i%20_000 == 0 {
-			logger.Log(lvl, "Generating idx for "+sn.Type.Name(), "progress", i)
+			logger.Log(lvl, "Generating idx for "+sn.Name(), "progress", i)
 		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)

From a9ed413d2d5f7fa3d09a1a67d8ba1d4bbb18fa2f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:54:48 +0200
Subject: [PATCH 012/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index dc9fcc06d91..a8b3d9bffe3 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -515,7 +515,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 			}
 			// keep beaconblocks here but whatever....
 			to := chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil)
-			logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to)
+			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
 			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, salt, dirs, workers, lvl, logger); err != nil {
 				return err
 			}

From aa960d93573c33a0b1330aff91d925014110f865 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:57:23 +0200
Subject: [PATCH 013/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index a8b3d9bffe3..526846fe47d 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -466,7 +466,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		return fmt.Errorf("expected %d blocks, got %d", snaptype.CaplinMergeLimit, sn.Count())
 	}
 	if err := sn.Compress(); err != nil {
-		return fmt.Errorf("compress: %w", err)
+		return err
 	}
 	// Generate .idx file, which is the slot => offset mapping.
 	p := &background.Progress{}

From b4801e20e70aa07ff5be94ff8a029e3202fbbd9d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 02:57:55 +0200
Subject: [PATCH 014/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 526846fe47d..ee21bdf27ce 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -458,7 +458,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		if i%20_000 == 0 {
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
 		}
-		if err := sn.AddWord(dump); err != nil {
+		if err := sn.AddUncompressedWord(dump); err != nil {
 			return err
 		}
 	}

From 8b606a67c2ef87ac7df260eecbdc8f70f9d7c5d0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 03:41:09 +0200
Subject: [PATCH 015/131] save

---
 cmd/capcli/cli.go                                     |  2 +-
 .../freezeblocks/caplin_state_snapshots.go            | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 4141b635a7b..a943e88695c 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -1214,7 +1214,7 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
 	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
 	stateSn.OpenFolder()
-	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
+	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err
 	}
 	return nil
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index ee21bdf27ce..7582e261ca9 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -504,17 +504,16 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st
 	return nil
 }
 
-func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
-	cfg := snapcfg.KnownCfg("")
+func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
 	for snapName, kvGetter := range s.snapshotTypes.Types {
-		for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) {
-			blocksPerFile := snapcfg.MergeLimitFromCfg(cfg, snaptype.CaplinEnums.BeaconBlocks, i)
-
+		fromSlot := fromSlot / blocksPerFile
+		toSlot := toSlot / blocksPerFile
+		for i := fromSlot; i < toSlot; i += blocksPerFile {
 			if toSlot-i < blocksPerFile {
 				break
 			}
 			// keep beaconblocks here but whatever....
-			to := chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil)
+			to := i
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
 			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, salt, dirs, workers, lvl, logger); err != nil {
 				return err

From 0588cae08ae0cf15fa89f77247de2fa1ff0ebad7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 03:42:55 +0200
Subject: [PATCH 016/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 7582e261ca9..ff7bad9f638 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -505,9 +505,10 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st
 }
 
 func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+	fromSlot = fromSlot / blocksPerFile
+	toSlot = toSlot / blocksPerFile
+	fmt.Println("DumpCaplinState", fromSlot, toSlot)
 	for snapName, kvGetter := range s.snapshotTypes.Types {
-		fromSlot := fromSlot / blocksPerFile
-		toSlot := toSlot / blocksPerFile
 		for i := fromSlot; i < toSlot; i += blocksPerFile {
 			if toSlot-i < blocksPerFile {
 				break

From d8d972ba3fde8166a178cd0eee09481159f8a4eb Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 03:43:48 +0200
Subject: [PATCH 017/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index ff7bad9f638..8a4dd9b6e24 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -505,8 +505,8 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st
 }
 
 func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
-	fromSlot = fromSlot / blocksPerFile
-	toSlot = toSlot / blocksPerFile
+	fromSlot = (fromSlot / blocksPerFile) * blocksPerFile
+	toSlot = (toSlot / blocksPerFile) * blocksPerFile
 	fmt.Println("DumpCaplinState", fromSlot, toSlot)
 	for snapName, kvGetter := range s.snapshotTypes.Types {
 		for i := fromSlot; i < toSlot; i += blocksPerFile {

From a98bd8441fe085d8267ff3bbf77c3ffbff69310b Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 03:44:24 +0200
Subject: [PATCH 018/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 8a4dd9b6e24..da08e485e08 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -514,7 +514,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 				break
 			}
 			// keep beaconblocks here but whatever....
-			to := i
+			to := i + blocksPerFile
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
 			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, salt, dirs, workers, lvl, logger); err != nil {
 				return err

From d51b414e70c230ef29cffa1ca69fbad561af3021 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 03:45:57 +0200
Subject: [PATCH 019/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index da08e485e08..c1f23a8df49 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -432,7 +432,7 @@ func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegme
 	return nil, false
 }
 
-func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
 	tmpDir, snapDir := dirs.Tmp, dirs.SnapCaplin
 
 	segName := snaptype.BeaconBlocks.FileName(0, fromSlot, toSlot)
@@ -462,8 +462,8 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 			return err
 		}
 	}
-	if sn.Count() != snaptype.CaplinMergeLimit {
-		return fmt.Errorf("expected %d blocks, got %d", snaptype.CaplinMergeLimit, sn.Count())
+	if sn.Count() != int(blocksPerFile) {
+		return fmt.Errorf("expected %d blocks, got %d", blocksPerFile, sn.Count())
 	}
 	if err := sn.Compress(); err != nil {
 		return err
@@ -516,7 +516,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 			// keep beaconblocks here but whatever....
 			to := i + blocksPerFile
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
-			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, salt, dirs, workers, lvl, logger); err != nil {
+			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, blocksPerFile, salt, dirs, workers, lvl, logger); err != nil {
 				return err
 			}
 		}

From 3749e485da7fd916290db862eb97fd2ac55af843 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 16:43:40 +0200
Subject: [PATCH 020/131] save

---
 cmd/capcli/cli.go                             |  2 ++
 .../freezeblocks/caplin_state_snapshots.go    | 36 ++++++++++++++++++-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a943e88695c..6de334f75ec 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -1214,6 +1214,8 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
 	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
 	stateSn.OpenFolder()
+	fmt.Println(stateSn.BlocksAvailable())
+	panic("A")
 	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err
 	}
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index c1f23a8df49..b713a71c774 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -267,7 +267,7 @@ Loop:
 			// then make segment available even if index open may fail
 			segments.DirtySegments.Set(sn)
 		}
-		if err := sn.openIdxIfNeed(s.dir, optimistic); err != nil {
+		if err := openIdxForCaplinStateIfNeeded(sn, s.dir, optimistic); err != nil {
 			return err
 		}
 		// Only bob sidecars count for progression
@@ -288,6 +288,40 @@ Loop:
 	return nil
 }
 
+func openIdxForCaplinStateIfNeeded(s *DirtySegment, dir string, optimistic bool) error {
+	if s.Decompressor == nil {
+		return nil
+	}
+	err := openIdxIfNeedForCaplinState(s, dir)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			if optimistic {
+				log.Warn("[snapshots] open index", "err", err)
+			} else {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func openIdxIfNeedForCaplinState(s *DirtySegment, dir string) (err error) {
+	s.indexes = make([]*recsplit.Index, 1)
+	if s.indexes[0] != nil {
+		return nil
+	}
+
+	fileName := strings.ReplaceAll(s.FileName(), ".seg", ".idx")
+	index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
+	if err != nil {
+		return fmt.Errorf("%w, fileName: %s", err, fileName)
+	}
+
+	s.indexes[0] = index
+
+	return nil
+}
 func (s *CaplinStateSnapshots) recalcVisibleFiles() {
 	defer func() {
 		s.idxMax.Store(s.idxAvailability())

From 29be4918d4cfe74a80a1f154d19101936a3f779f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 16:46:24 +0200
Subject: [PATCH 021/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index b713a71c774..e271b9d6aac 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -247,6 +247,7 @@ Loop:
 			}
 		}
 		if err := sn.openSegIfNeed(s.dir); err != nil {
+			fmt.Println(err)
 			if errors.Is(err, os.ErrNotExist) {
 				if optimistic {
 					continue Loop
@@ -268,6 +269,7 @@ Loop:
 			segments.DirtySegments.Set(sn)
 		}
 		if err := openIdxForCaplinStateIfNeeded(sn, s.dir, optimistic); err != nil {
+			fmt.Println(err)
 			return err
 		}
 		// Only bob sidecars count for progression

From 1bc75ac7a40fa49d1d181008171eeb905d024aec Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 16:48:27 +0200
Subject: [PATCH 022/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index e271b9d6aac..d5fffc527f8 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -214,6 +214,7 @@ func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) err
 Loop:
 	for _, fName := range fileNames {
 		f, _, ok := snaptype.ParseFileName(s.dir, fName)
+		fmt.Println(f, ok)
 		if !ok {
 			continue
 		}

From 207138188b8258b2d7122e566a3feb7de4c05cca Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 16:52:25 +0200
Subject: [PATCH 023/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index d5fffc527f8..35de19d5a5b 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -204,6 +204,7 @@ func (s *CaplinStateSnapshots) Close() {
 // OpenList stops on optimistic=false, continue opening files on optimistic=true
 func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) error {
 	defer s.recalcVisibleFiles()
+	fmt.Println(fileNames)
 
 	s.dirtySegmentsLock.Lock()
 	defer s.dirtySegmentsLock.Unlock()

From b390a57a76b56e34fbdfffb22756d891df722638 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:01:29 +0200
Subject: [PATCH 024/131] save

---
 .../freezeblocks/caplin_state_snapshots.go    | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 35de19d5a5b..2187e0cf832 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -383,17 +383,26 @@ func (s *CaplinStateSnapshots) idxAvailability() uint64 {
 	return minVisible
 }
 
-func (s *CaplinStateSnapshots) OpenFolder() error {
-	files, _, err := SegmentsCaplin(s.dir, s.segmentsMin.Load())
+func listAllSegFilesInDir(dir string) []string {
+	files, err := os.ReadDir(dir)
 	if err != nil {
-		return err
+		panic(err)
 	}
 	list := make([]string, 0, len(files))
 	for _, f := range files {
-		_, fName := filepath.Split(f.Path)
-		list = append(list, fName)
+		if f.IsDir() {
+			continue
+		}
+		// check if it's a .seg file
+		if filepath.Ext(f.Name()) != ".seg" {
+			continue
+		}
 	}
-	return s.OpenList(list, false)
+	return list
+}
+
+func (s *CaplinStateSnapshots) OpenFolder() error {
+	return s.OpenList(listAllSegFilesInDir(s.dir), false)
 }
 
 func (s *CaplinStateSnapshots) closeWhatNotInList(l []string) {

From 193967abcc9a7a71d4ae91e17f8671ac7b60e0a0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:02:40 +0200
Subject: [PATCH 025/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 2187e0cf832..c15ff23e063 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -397,6 +397,7 @@ func listAllSegFilesInDir(dir string) []string {
 		if filepath.Ext(f.Name()) != ".seg" {
 			continue
 		}
+		list = append(list, f.Name())
 	}
 	return list
 }

From 1f359a6ca305b0b876c82843ea1b63ed27db4ee3 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:05:07 +0200
Subject: [PATCH 026/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index c15ff23e063..708beb5deaf 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -214,15 +214,13 @@ func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) err
 	var segmentsMaxSet bool
 Loop:
 	for _, fName := range fileNames {
-		f, _, ok := snaptype.ParseFileName(s.dir, fName)
-		fmt.Println(f, ok)
-		if !ok {
-			continue
-		}
+		f, _, _ := snaptype.ParseFileName(s.dir, fName)
 
 		var processed bool = true
 		var exists bool
 		var sn *DirtySegment
+
+		fmt.Println(f.TypeString)
 		segments, ok := s.Segments[f.TypeString]
 		if !ok {
 			continue

From 52d3d1fa3c928128dd7d8f0a690ab996e49a82f5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:06:00 +0200
Subject: [PATCH 027/131] save

---
 erigon-lib/downloader/snaptype/files.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go
index 2fd09861fe9..4ac92207dc2 100644
--- a/erigon-lib/downloader/snaptype/files.go
+++ b/erigon-lib/downloader/snaptype/files.go
@@ -152,11 +152,12 @@ func parseFileName(dir, fileName string) (res FileInfo, ok bool) {
 		return
 	}
 	res.To = to * 1_000
+	res.TypeString = parts[3]
+
 	res.Type, ok = ParseFileType(parts[3])
 	if !ok {
 		return res, ok
 	}
-	res.TypeString = parts[3]
 	return res, ok
 }
 

From 8ffef872e602d2baa8b1fd34046deab68cd37c02 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:08:43 +0200
Subject: [PATCH 028/131] save

---
 .../freezeblocks/caplin_state_snapshots.go        | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 708beb5deaf..32339f30985 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -201,6 +201,17 @@ func (s *CaplinStateSnapshots) Close() {
 	s.closeWhatNotInList(nil)
 }
 
+func (s *CaplinStateSnapshots) openSegIfNeed(sn *DirtySegment, dir string) error {
+	if sn.Decompressor != nil {
+		return nil
+	}
+	var err error
+	sn.Decompressor, err = seg.NewDecompressor(sn.FilePath())
+	if err != nil {
+		return fmt.Errorf("%w, fileName: %s", err, sn.FilePath())
+	}
+}
+
 // OpenList stops on optimistic=false, continue opening files on optimistic=true
 func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) error {
 	defer s.recalcVisibleFiles()
@@ -220,7 +231,6 @@ Loop:
 		var exists bool
 		var sn *DirtySegment
 
-		fmt.Println(f.TypeString)
 		segments, ok := s.Segments[f.TypeString]
 		if !ok {
 			continue
@@ -246,7 +256,8 @@ Loop:
 				frozen:  snapcfg.IsFrozen(s.cfg.ChainName, f),
 			}
 		}
-		if err := sn.openSegIfNeed(s.dir); err != nil {
+
+		if err := s.openSegIfNeed(sn, s.dir); err != nil {
 			fmt.Println(err)
 			if errors.Is(err, os.ErrNotExist) {
 				if optimistic {

From ea95bc300738e6821b878440a79451f07a80aa74 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:09:06 +0200
Subject: [PATCH 029/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 32339f30985..64c06e43e2d 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -210,6 +210,7 @@ func (s *CaplinStateSnapshots) openSegIfNeed(sn *DirtySegment, dir string) error
 	if err != nil {
 		return fmt.Errorf("%w, fileName: %s", err, sn.FilePath())
 	}
+	return nil
 }
 
 // OpenList stops on optimistic=false, continue opening files on optimistic=true

From 2e7f829a14d8287c348724255165143957b84c8d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:10:47 +0200
Subject: [PATCH 030/131] save

---
 .../freezeblocks/caplin_state_snapshots.go             | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 64c06e43e2d..9c83b7202c3 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -201,14 +201,14 @@ func (s *CaplinStateSnapshots) Close() {
 	s.closeWhatNotInList(nil)
 }
 
-func (s *CaplinStateSnapshots) openSegIfNeed(sn *DirtySegment, dir string) error {
+func (s *CaplinStateSnapshots) openSegIfNeed(sn *DirtySegment, filepath string) error {
 	if sn.Decompressor != nil {
 		return nil
 	}
 	var err error
-	sn.Decompressor, err = seg.NewDecompressor(sn.FilePath())
+	sn.Decompressor, err = seg.NewDecompressor(filepath)
 	if err != nil {
-		return fmt.Errorf("%w, fileName: %s", err, sn.FilePath())
+		return fmt.Errorf("%w, fileName: %s", err, filepath)
 	}
 	return nil
 }
@@ -257,8 +257,8 @@ Loop:
 				frozen:  snapcfg.IsFrozen(s.cfg.ChainName, f),
 			}
 		}
-
-		if err := s.openSegIfNeed(sn, s.dir); err != nil {
+		filePath := filepath.Join(s.dir, fName)
+		if err := s.openSegIfNeed(sn, filePath); err != nil {
 			fmt.Println(err)
 			if errors.Is(err, os.ErrNotExist) {
 				if optimistic {

From 54238620fe1d9bea77a708295a03726de52f57d8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:12:52 +0200
Subject: [PATCH 031/131] save

---
 .../freezeblocks/caplin_state_snapshots.go       | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 9c83b7202c3..546d81e05b6 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -336,6 +336,20 @@ func openIdxIfNeedForCaplinState(s *DirtySegment, dir string) (err error) {
 
 	return nil
 }
+
+func isIndexed(s *DirtySegment) bool {
+	if s.Decompressor == nil {
+		return false
+	}
+
+	for _, idx := range s.indexes {
+		if idx == nil {
+			return false
+		}
+	}
+	return true
+}
+
 func (s *CaplinStateSnapshots) recalcVisibleFiles() {
 	defer func() {
 		s.idxMax.Store(s.idxAvailability())
@@ -352,7 +366,7 @@ func (s *CaplinStateSnapshots) recalcVisibleFiles() {
 				if sn.canDelete.Load() {
 					continue
 				}
-				if !sn.Indexed() {
+				if !isIndexed(sn) {
 					continue
 				}
 				for len(newVisibleSegments) > 0 && newVisibleSegments[len(newVisibleSegments)-1].src.isSubSetOf(sn) {

From 4a7282051b49c4982822cf8782cf3c0a06d375f6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:15:04 +0200
Subject: [PATCH 032/131] save

---
 .../freezeblocks/caplin_state_snapshots.go         | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 546d81e05b6..ab78af4540b 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -280,7 +280,7 @@ Loop:
 			// then make segment available even if index open may fail
 			segments.DirtySegments.Set(sn)
 		}
-		if err := openIdxForCaplinStateIfNeeded(sn, s.dir, optimistic); err != nil {
+		if err := openIdxForCaplinStateIfNeeded(sn, filePath, optimistic); err != nil {
 			fmt.Println(err)
 			return err
 		}
@@ -302,11 +302,11 @@ Loop:
 	return nil
 }
 
-func openIdxForCaplinStateIfNeeded(s *DirtySegment, dir string, optimistic bool) error {
+func openIdxForCaplinStateIfNeeded(s *DirtySegment, filePath string, optimistic bool) error {
 	if s.Decompressor == nil {
 		return nil
 	}
-	err := openIdxIfNeedForCaplinState(s, dir)
+	err := openIdxIfNeedForCaplinState(s, filePath)
 	if err != nil {
 		if !errors.Is(err, os.ErrNotExist) {
 			if optimistic {
@@ -320,16 +320,16 @@ func openIdxForCaplinStateIfNeeded(s *DirtySegment, dir string, optimistic bool)
 	return nil
 }
 
-func openIdxIfNeedForCaplinState(s *DirtySegment, dir string) (err error) {
+func openIdxIfNeedForCaplinState(s *DirtySegment, filePath string) (err error) {
 	s.indexes = make([]*recsplit.Index, 1)
 	if s.indexes[0] != nil {
 		return nil
 	}
 
-	fileName := strings.ReplaceAll(s.FileName(), ".seg", ".idx")
-	index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
+	filePath = strings.ReplaceAll(filePath, ".seg", ".idx")
+	index, err := recsplit.OpenIndex(filepath.Join(dir, filePath))
 	if err != nil {
-		return fmt.Errorf("%w, fileName: %s", err, fileName)
+		return fmt.Errorf("%w, fileName: %s", err, filePath)
 	}
 
 	s.indexes[0] = index

From 60317d91b2eee1cc0d6dde98bef1b7920d680e00 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:15:28 +0200
Subject: [PATCH 033/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index ab78af4540b..73e57f4ed21 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -327,7 +327,7 @@ func openIdxIfNeedForCaplinState(s *DirtySegment, filePath string) (err error) {
 	}
 
 	filePath = strings.ReplaceAll(filePath, ".seg", ".idx")
-	index, err := recsplit.OpenIndex(filepath.Join(dir, filePath))
+	index, err := recsplit.OpenIndex(filePath)
 	if err != nil {
 		return fmt.Errorf("%w, fileName: %s", err, filePath)
 	}

From 6c12a91b6f71c6cf4b1de59143a7bb619b12482d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:19:14 +0200
Subject: [PATCH 034/131] save

---
 cmd/capcli/cli.go                                      | 10 ++++++++++
 .../freezeblocks/caplin_state_snapshots.go             |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 6de334f75ec..4237eaa92aa 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -17,6 +17,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -1214,6 +1215,15 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
 	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
 	stateSn.OpenFolder()
+	v, err := stateSn.Get(kv.SlotData, 100)
+	if err != nil {
+		return err
+	}
+	buf := bytes.NewBuffer(v)
+	r := &state_accessors.SlotData{}
+	if err := r.ReadFrom(buf); err != nil {
+		return err
+	}
 	fmt.Println(stateSn.BlocksAvailable())
 	panic("A")
 	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 73e57f4ed21..3fd893d85d0 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -644,7 +644,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	var buf []byte
 
-	seg, ok := view.VisibleSegment(slot, tbl)
+	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
 		return nil, nil
 	}

From e772766557622ec351255374aad1e4d246a64639 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:20:50 +0200
Subject: [PATCH 035/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 3fd893d85d0..cbbfcdf0394 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -468,7 +468,7 @@ func (s *CaplinStateSnapshots) View() *CaplinStateView {
 	s.visibleSegmentsLock.RLock()
 	defer s.visibleSegmentsLock.RUnlock()
 
-	v := &CaplinStateView{s: s}
+	v := &CaplinStateView{s: s, roTxs: make(map[string]*segmentsRotx)}
 	// BeginRo increments refcount - which is contended
 	s.dirtySegmentsLock.RLock()
 	defer s.dirtySegmentsLock.RUnlock()
@@ -635,7 +635,7 @@ func (s *CaplinStateSnapshots) BuildMissingIndices(ctx context.Context, logger l
 func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			panic(fmt.Sprintf("ReadHeader(%d), %s, %s\n", slot, rec, dbg.Stack()))
+			panic(fmt.Sprintf("Get(%s, %d), %s, %s\n", tbl, slot, rec, dbg.Stack()))
 		}
 	}()
 

From 904248bec75a1ee48e91d6857cfc2863debc9f37 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:21:42 +0200
Subject: [PATCH 036/131] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 4237eaa92aa..e036691183c 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -1224,7 +1224,7 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	if err := r.ReadFrom(buf); err != nil {
 		return err
 	}
-	fmt.Println(stateSn.BlocksAvailable())
+	fmt.Println(stateSn.BlocksAvailable(), r.ValidatorLength)
 	panic("A")
 	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err

From e123bf7698d5c579f1d9459bea559721af533d00 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:44:42 +0200
Subject: [PATCH 037/131] save

---
 cmd/capcli/cli.go | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index e036691183c..5c1d0cf0f95 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -17,7 +17,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -1214,18 +1213,10 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	}
 	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
 	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
-	stateSn.OpenFolder()
-	v, err := stateSn.Get(kv.SlotData, 100)
-	if err != nil {
-		return err
-	}
-	buf := bytes.NewBuffer(v)
-	r := &state_accessors.SlotData{}
-	if err := r.ReadFrom(buf); err != nil {
+	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
-	fmt.Println(stateSn.BlocksAvailable(), r.ValidatorLength)
-	panic("A")
+
 	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err
 	}

From df8e2ef2369deee481b1d6e422a4691687c1f8a0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 17:45:02 +0200
Subject: [PATCH 038/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index cbbfcdf0394..813043149e9 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -216,7 +216,6 @@ func (s *CaplinStateSnapshots) openSegIfNeed(sn *DirtySegment, filepath string)
 // OpenList stops on optimistic=false, continue opening files on optimistic=true
 func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) error {
 	defer s.recalcVisibleFiles()
-	fmt.Println(fileNames)
 
 	s.dirtySegmentsLock.Lock()
 	defer s.dirtySegmentsLock.Unlock()

From 22e256521de7c5ff3abed5ecb62639447f0d0633 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 18:15:36 +0200
Subject: [PATCH 039/131] save

---
 cl/antiquary/antiquary.go                     |  4 +-
 cl/antiquary/state_antiquary.go               |  2 +-
 cl/beacon/handler/attestation_rewards.go      |  6 +-
 cl/beacon/handler/committees.go               |  4 +-
 cl/beacon/handler/duties_attester.go          |  5 +-
 cl/beacon/handler/duties_sync.go              |  4 +-
 cl/beacon/handler/handler.go                  | 51 +++++++++--------
 cl/beacon/handler/lighthouse.go               | 10 ++--
 cl/beacon/handler/rewards.go                  |  8 ++-
 cl/beacon/handler/states.go                   |  9 ++-
 .../attesting_indicies.go                     |  4 +-
 .../historical_states_reader.go               | 56 ++++++++++++-------
 cl/persistence/state/state_accessors.go       | 43 ++++++++++----
 cmd/caplin/caplin1/run.go                     |  7 ++-
 .../freezeblocks/caplin_state_snapshots.go    |  2 +
 15 files changed, 137 insertions(+), 78 deletions(-)

diff --git a/cl/antiquary/antiquary.go b/cl/antiquary/antiquary.go
index 7f49882bfcd..806ccbf4bc2 100644
--- a/cl/antiquary/antiquary.go
+++ b/cl/antiquary/antiquary.go
@@ -50,6 +50,7 @@ type Antiquary struct {
 	downloader                     proto_downloader.DownloaderClient
 	logger                         log.Logger
 	sn                             *freezeblocks.CaplinSnapshots
+	stateSn                        *freezeblocks.CaplinStateSnapshots
 	snReader                       freezeblocks.BeaconSnapshotReader
 	snBuildSema                    *semaphore.Weighted // semaphore for building only one type (blocks, caplin, v3) at a time
 	ctx                            context.Context
@@ -65,7 +66,7 @@ type Antiquary struct {
 	balances32   []byte
 }
 
-func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
+func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, stateSn *freezeblocks.CaplinStateSnapshots, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
 	backfilled := &atomic.Bool{}
 	blobBackfilled := &atomic.Bool{}
 	backfilled.Store(false)
@@ -89,6 +90,7 @@ func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, gen
 		blocks:          blocks,
 		blobs:           blobs,
 		snapgen:         snapgen,
+		stateSn:         stateSn,
 	}
 }
 
diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index efb5238954b..92684897495 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -444,7 +444,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 	backoffStrides := uint64(10)
 	backoffStep := backoffStrides
 
-	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState)
+	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState, s.stateSn)
 
 	for {
 		attempt, err := computeSlotToBeRequested(tx, s.cfg, s.genesisState.Slot(), targetSlot, backoffStep)
diff --git a/cl/beacon/handler/attestation_rewards.go b/cl/beacon/handler/attestation_rewards.go
index 364340a6e4c..4c09934754a 100644
--- a/cl/beacon/handler/attestation_rewards.go
+++ b/cl/beacon/handler/attestation_rewards.go
@@ -179,7 +179,9 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("requested range is not yet processed or the node is not archivial"))
 	}
 
-	epochData, err := state_accessors.ReadEpochData(tx, a.beaconChainCfg.RoundSlotToEpoch(lastSlot))
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+
+	epochData, err := state_accessors.ReadEpochData(stateGetter, a.beaconChainCfg.RoundSlotToEpoch(lastSlot))
 	if err != nil {
 		return nil, err
 	}
@@ -197,7 +199,7 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 		return nil, err
 	}
 
-	_, _, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	_, _, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/committees.go b/cl/beacon/handler/committees.go
index 8c2db31ffea..6074879d492 100644
--- a/cl/beacon/handler/committees.go
+++ b/cl/beacon/handler/committees.go
@@ -120,7 +120,9 @@ func (a *ApiHandler) getCommittees(w http.ResponseWriter, r *http.Request) (*bea
 		return newBeaconResponse(resp).WithFinalized(isFinalized).WithOptimistic(isOptimistic), nil
 	}
 	// finality case
-	activeIdxs, err := state_accessors.ReadActiveIndicies(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	activeIdxs, err := state_accessors.ReadActiveIndicies(
+		state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+		epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/duties_attester.go b/cl/beacon/handler/duties_attester.go
index 604807e50ba..8452fbf3332 100644
--- a/cl/beacon/handler/duties_attester.go
+++ b/cl/beacon/handler/duties_attester.go
@@ -149,8 +149,11 @@ func (a *ApiHandler) getAttesterDuties(w http.ResponseWriter, r *http.Request) (
 	if (epoch)*a.beaconChainCfg.SlotsPerEpoch >= stageStateProgress {
 		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, fmt.Errorf("epoch %d is too far in the future", epoch))
 	}
+
 	// finality case
-	activeIdxs, err := state_accessors.ReadActiveIndicies(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	activeIdxs, err := state_accessors.ReadActiveIndicies(
+		state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+		epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/duties_sync.go b/cl/beacon/handler/duties_sync.go
index 024fd6d45e5..2609d0b3103 100644
--- a/cl/beacon/handler/duties_sync.go
+++ b/cl/beacon/handler/duties_sync.go
@@ -83,7 +83,9 @@ func (a *ApiHandler) getSyncDuties(w http.ResponseWriter, r *http.Request) (*bea
 	}
 	// Read them from the archive node if we do not have them in the fast-access storage
 	if !ok {
-		syncCommittee, err = state_accessors.ReadCurrentSyncCommittee(tx, a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(startSlotAtEpoch))
+		syncCommittee, err = state_accessors.ReadCurrentSyncCommittee(
+			state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+			a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(startSlotAtEpoch))
 		if syncCommittee == nil {
 			log.Warn("could not find sync committee for epoch", "epoch", epoch, "period", period)
 			return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not find sync committee for epoch %d", epoch))
diff --git a/cl/beacon/handler/handler.go b/cl/beacon/handler/handler.go
index 093fa6ada1b..fe2e1be10fe 100644
--- a/cl/beacon/handler/handler.go
+++ b/cl/beacon/handler/handler.go
@@ -63,18 +63,19 @@ type ApiHandler struct {
 	o   sync.Once
 	mux *chi.Mux
 
-	blockReader     freezeblocks.BeaconSnapshotReader
-	indiciesDB      kv.RwDB
-	netConfig       *clparams.NetworkConfig
-	ethClock        eth_clock.EthereumClock
-	beaconChainCfg  *clparams.BeaconChainConfig
-	forkchoiceStore forkchoice.ForkChoiceStorage
-	operationsPool  pool.OperationsPool
-	syncedData      synced_data.SyncedData
-	stateReader     *historical_states_reader.HistoricalStatesReader
-	sentinel        sentinel.SentinelClient
-	blobStoage      blob_storage.BlobStorage
-	caplinSnapshots *freezeblocks.CaplinSnapshots
+	blockReader          freezeblocks.BeaconSnapshotReader
+	indiciesDB           kv.RwDB
+	netConfig            *clparams.NetworkConfig
+	ethClock             eth_clock.EthereumClock
+	beaconChainCfg       *clparams.BeaconChainConfig
+	forkchoiceStore      forkchoice.ForkChoiceStorage
+	operationsPool       pool.OperationsPool
+	syncedData           synced_data.SyncedData
+	stateReader          *historical_states_reader.HistoricalStatesReader
+	sentinel             sentinel.SentinelClient
+	blobStoage           blob_storage.BlobStorage
+	caplinSnapshots      *freezeblocks.CaplinSnapshots
+	caplinStateSnapshots *freezeblocks.CaplinStateSnapshots
 
 	version string // Node's version
 
@@ -141,24 +142,26 @@ func NewApiHandler(
 	proposerSlashingService services.ProposerSlashingService,
 	builderClient builder.BuilderClient,
 	validatorMonitor monitor.ValidatorMonitor,
+	caplinStateSnapshots *freezeblocks.CaplinStateSnapshots,
 ) *ApiHandler {
 	blobBundles, err := lru.New[common.Bytes48, BlobBundle]("blobs", maxBlobBundleCacheSize)
 	if err != nil {
 		panic(err)
 	}
 	return &ApiHandler{
-		logger:          logger,
-		validatorParams: validatorParams,
-		o:               sync.Once{},
-		netConfig:       netConfig,
-		ethClock:        ethClock,
-		beaconChainCfg:  beaconChainConfig,
-		indiciesDB:      indiciesDB,
-		forkchoiceStore: forkchoiceStore,
-		operationsPool:  operationsPool,
-		blockReader:     rcsn,
-		syncedData:      syncedData,
-		stateReader:     stateReader,
+		logger:               logger,
+		validatorParams:      validatorParams,
+		o:                    sync.Once{},
+		netConfig:            netConfig,
+		ethClock:             ethClock,
+		beaconChainCfg:       beaconChainConfig,
+		indiciesDB:           indiciesDB,
+		forkchoiceStore:      forkchoiceStore,
+		operationsPool:       operationsPool,
+		blockReader:          rcsn,
+		syncedData:           syncedData,
+		stateReader:          stateReader,
+		caplinStateSnapshots: caplinStateSnapshots,
 		randaoMixesPool: sync.Pool{New: func() interface{} {
 			return solid.NewHashVector(int(beaconChainConfig.EpochsPerHistoricalVector))
 		}},
diff --git a/cl/beacon/handler/lighthouse.go b/cl/beacon/handler/lighthouse.go
index 612cb31e480..10baeca3888 100644
--- a/cl/beacon/handler/lighthouse.go
+++ b/cl/beacon/handler/lighthouse.go
@@ -75,6 +75,7 @@ func (a *ApiHandler) GetLighthouseValidatorInclusionGlobal(w http.ResponseWriter
 		return nil, err
 	}
 	defer tx.Rollback()
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
 
 	slot := epoch * a.beaconChainCfg.SlotsPerEpoch
 	if slot >= a.forkchoiceStore.LowestAvailableSlot() {
@@ -120,14 +121,14 @@ func (a *ApiHandler) GetLighthouseValidatorInclusionGlobal(w http.ResponseWriter
 	}
 
 	// read the epoch datas first
-	epochData, err := state_accessors.ReadEpochData(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	epochData, err := state_accessors.ReadEpochData(stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
 	if epochData == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("epoch data not found for current epoch"))
 	}
-	prevEpochData, err := state_accessors.ReadEpochData(tx, prevEpoch*a.beaconChainCfg.SlotsPerEpoch)
+	prevEpochData, err := state_accessors.ReadEpochData(stateGetter, prevEpoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
@@ -277,15 +278,16 @@ func (a *ApiHandler) GetLighthouseValidatorInclusion(w http.ResponseWriter, r *h
 		return newBeaconResponse(a.computeLighthouseValidatorInclusion(int(validatorIndex), prevEpoch, epoch, activeBalance, prevActiveBalance, validatorSet, currentEpochParticipation, previousEpochParticipation)), nil
 	}
 
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
 	// read the epoch datas first
-	epochData, err := state_accessors.ReadEpochData(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	epochData, err := state_accessors.ReadEpochData(stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
 	if epochData == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("epoch data not found for current epoch"))
 	}
-	prevEpochData, err := state_accessors.ReadEpochData(tx, prevEpoch*a.beaconChainCfg.SlotsPerEpoch)
+	prevEpochData, err := state_accessors.ReadEpochData(stateGetter, prevEpoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/rewards.go b/cl/beacon/handler/rewards.go
index bec4923de39..aacf6d90307 100644
--- a/cl/beacon/handler/rewards.go
+++ b/cl/beacon/handler/rewards.go
@@ -81,7 +81,8 @@ func (a *ApiHandler) GetEthV1BeaconRewardsBlocks(w http.ResponseWriter, r *http.
 			Total:             blkRewards.Attestations + blkRewards.ProposerSlashings + blkRewards.AttesterSlashings + blkRewards.SyncAggregate,
 		}).WithFinalized(isFinalized).WithOptimistic(isOptimistic), nil
 	}
-	slotData, err := state_accessors.ReadSlotData(tx, slot)
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+	slotData, err := state_accessors.ReadSlotData(stateGetter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -165,11 +166,12 @@ func (a *ApiHandler) PostEthV1BeaconRewardsSyncCommittees(w http.ResponseWriter,
 		syncCommittee      *solid.SyncCommittee
 		totalActiveBalance uint64
 	)
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
 	if slot < a.forkchoiceStore.LowestAvailableSlot() {
 		if !isCanonical {
 			return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("non-canonical finalized block not found"))
 		}
-		epochData, err := state_accessors.ReadEpochData(tx, a.beaconChainCfg.RoundSlotToEpoch(blk.Block.Slot))
+		epochData, err := state_accessors.ReadEpochData(getter, a.beaconChainCfg.RoundSlotToEpoch(blk.Block.Slot))
 		if err != nil {
 			return nil, err
 		}
@@ -177,7 +179,7 @@ func (a *ApiHandler) PostEthV1BeaconRewardsSyncCommittees(w http.ResponseWriter,
 			return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("could not read historical sync committee rewards, node may not be archive or it still processing historical states"))
 		}
 		totalActiveBalance = epochData.TotalActiveBalance
-		syncCommittee, err = state_accessors.ReadCurrentSyncCommittee(tx, a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(blk.Block.Slot))
+		syncCommittee, err = state_accessors.ReadCurrentSyncCommittee(getter, a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(blk.Block.Slot))
 		if err != nil {
 			return nil, err
 		}
diff --git a/cl/beacon/handler/states.go b/cl/beacon/handler/states.go
index 011a9e43687..d5f8c66c6d7 100644
--- a/cl/beacon/handler/states.go
+++ b/cl/beacon/handler/states.go
@@ -262,8 +262,10 @@ func (a *ApiHandler) getFinalityCheckpoints(w http.ResponseWriter, r *http.Reque
 	if err != nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
 	}
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
 	if !ok {
-		currentJustifiedCheckpoint, previousJustifiedCheckpoint, finalizedCheckpoint, ok, err = state_accessors.ReadCheckpoints(tx, a.beaconChainCfg.RoundSlotToEpoch(*slot))
+		currentJustifiedCheckpoint, previousJustifiedCheckpoint, finalizedCheckpoint, ok, err = state_accessors.ReadCheckpoints(stateGetter, a.beaconChainCfg.RoundSlotToEpoch(*slot))
 		if err != nil {
 			return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
 		}
@@ -316,17 +318,18 @@ func (a *ApiHandler) getSyncCommittees(w http.ResponseWriter, r *http.Request) (
 	if slot == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not read block slot: %x", blockRoot))
 	}
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
 
 	// Code here
 	currentSyncCommittee, nextSyncCommittee, ok := a.forkchoiceStore.GetSyncCommittees(a.beaconChainCfg.SyncCommitteePeriod(*slot))
 	if !ok {
 		syncCommitteeSlot := a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(*slot)
 		// Check the main database if it cannot be found in the forkchoice store
-		currentSyncCommittee, err = state_accessors.ReadCurrentSyncCommittee(tx, syncCommitteeSlot)
+		currentSyncCommittee, err = state_accessors.ReadCurrentSyncCommittee(stateGetter, syncCommitteeSlot)
 		if err != nil {
 			return nil, err
 		}
-		nextSyncCommittee, err = state_accessors.ReadNextSyncCommittee(tx, syncCommitteeSlot)
+		nextSyncCommittee, err = state_accessors.ReadNextSyncCommittee(stateGetter, syncCommitteeSlot)
 		if err != nil {
 			return nil, err
 		}
diff --git a/cl/persistence/state/historical_states_reader/attesting_indicies.go b/cl/persistence/state/historical_states_reader/attesting_indicies.go
index f0ef90b286c..d02ab0d7023 100644
--- a/cl/persistence/state/historical_states_reader/attesting_indicies.go
+++ b/cl/persistence/state/historical_states_reader/attesting_indicies.go
@@ -164,7 +164,9 @@ func (r *HistoricalStatesReader) readHistoricalBlockRoot(tx kv.Tx, slot, index u
 }
 
 func (r *HistoricalStatesReader) getAttestationParticipationFlagIndicies(tx kv.Tx, version clparams.StateVersion, stateSlot uint64, data solid.AttestationData, inclusionDelay uint64, skipAssert bool) ([]uint8, error) {
-	currentCheckpoint, previousCheckpoint, _, ok, err := state_accessors.ReadCheckpoints(tx, r.cfg.RoundSlotToEpoch(stateSlot))
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+
+	currentCheckpoint, previousCheckpoint, _, ok, err := state_accessors.ReadCheckpoints(getter, r.cfg.RoundSlotToEpoch(stateSlot))
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index d1cd90c670c..635390e9daa 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -46,6 +46,7 @@ type HistoricalStatesReader struct {
 	cfg            *clparams.BeaconChainConfig
 	validatorTable *state_accessors.StaticValidatorTable // We can save 80% of the I/O by caching the validator table
 	blockReader    freezeblocks.BeaconSnapshotReader
+	stateSn        *freezeblocks.CaplinStateSnapshots
 	genesisState   *state.CachingBeaconState
 
 	// cache for shuffled sets
@@ -56,7 +57,7 @@ func NewHistoricalStatesReader(
 	cfg *clparams.BeaconChainConfig,
 	blockReader freezeblocks.BeaconSnapshotReader,
 	validatorTable *state_accessors.StaticValidatorTable,
-	genesisState *state.CachingBeaconState) *HistoricalStatesReader {
+	genesisState *state.CachingBeaconState, stateSn *freezeblocks.CaplinStateSnapshots) *HistoricalStatesReader {
 
 	cache, err := lru.New[uint64, []uint64]("shuffledSetsCache_reader", 125)
 	if err != nil {
@@ -69,15 +70,19 @@ func NewHistoricalStatesReader(
 		genesisState:      genesisState,
 		validatorTable:    validatorTable,
 		shuffledSetsCache: cache,
+		stateSn:           stateSn,
 	}
 }
 
 func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.Tx, slot uint64) (*state.CachingBeaconState, error) {
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+
 	ret := state.New(r.cfg)
 	latestProcessedState, err := state_accessors.GetStateProcessingProgress(tx)
 	if err != nil {
 		return nil, err
 	}
+	latestProcessedState = max(latestProcessedState, r.stateSn.BlocksAvailable())
 
 	// If this happens, we need to update our static tables
 	if slot > latestProcessedState || slot > r.validatorTable.Slot() {
@@ -100,7 +105,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	blockHeader := block.SignedBeaconBlockHeader().Header
 	blockHeader.Root = common.Hash{}
 	// Read the epoch and per-slot data.
-	slotData, err := state_accessors.ReadSlotData(tx, slot)
+	slotData, err := state_accessors.ReadSlotData(getter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -110,7 +115,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	}
 	roundedSlot := r.cfg.RoundSlotToEpoch(slot)
 
-	epochData, err := state_accessors.ReadEpochData(tx, roundedSlot)
+	epochData, err := state_accessors.ReadEpochData(getter, roundedSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read epoch data: %w", err)
 	}
@@ -188,7 +193,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	ret.SetSlashings(slashingsVector)
 
 	// Finality
-	currentCheckpoint, previousCheckpoint, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(tx, roundedSlot)
+	currentCheckpoint, previousCheckpoint, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(getter, roundedSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read checkpoints: %w", err)
 	}
@@ -232,7 +237,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	ret.SetInactivityScoresRaw(inactivityScores)
 	// Sync
 	syncCommitteeSlot := r.cfg.RoundSlotToSyncCommitteePeriod(slot)
-	currentSyncCommittee, err := state_accessors.ReadCurrentSyncCommittee(tx, syncCommitteeSlot)
+	currentSyncCommittee, err := state_accessors.ReadCurrentSyncCommittee(getter, syncCommitteeSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read current sync committee: %w", err)
 	}
@@ -240,7 +245,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 		currentSyncCommittee = r.genesisState.CurrentSyncCommittee()
 	}
 
-	nextSyncCommittee, err := state_accessors.ReadNextSyncCommittee(tx, syncCommitteeSlot)
+	nextSyncCommittee, err := state_accessors.ReadNextSyncCommittee(getter, syncCommitteeSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read next sync committee: %w", err)
 	}
@@ -379,8 +384,10 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 		currKeyEpoch++
 		out.Set(int(currKeyEpoch%size), genesisVector.Get(int(currKeyEpoch%size)))
 	}
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+
 	// Now we need to read the intra epoch randao mix.
-	intraRandaoMix, err := tx.GetOne(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
+	intraRandaoMix, err := getter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return err
 	}
@@ -395,6 +402,7 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 	// Read the file
 	remainder := slot % clparams.SlotsPerDump
 	freshDumpSlot := slot - remainder
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	midpoint := uint64(clparams.SlotsPerDump / 2)
 	var compressed []byte
@@ -404,12 +412,12 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 	}
 	forward := remainder <= midpoint || currentStageProgress <= freshDumpSlot+clparams.SlotsPerDump
 	if forward {
-		compressed, err = tx.GetOne(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
+		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		compressed, err = tx.GetOne(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
+		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
 		if err != nil {
 			return nil, err
 		}
@@ -488,6 +496,7 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLength, slot uint64, diffBucket, dumpBucket string) ([]byte, error) {
 	remainder := slot % clparams.SlotsPerDump
 	freshDumpSlot := slot - remainder
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	buffer := buffersPool.Get().(*bytes.Buffer)
 	defer buffersPool.Put(buffer)
@@ -501,12 +510,12 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 	midpoint := uint64(clparams.SlotsPerDump / 2)
 	forward := remainder <= midpoint || currentStageProgress <= freshDumpSlot+clparams.SlotsPerDump
 	if forward {
-		compressed, err = tx.GetOne(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
+		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		compressed, err = tx.GetOne(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
+		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
 		if err != nil {
 			return nil, err
 		}
@@ -535,7 +544,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 			if i == freshDumpSlot {
 				continue
 			}
-			diff, err := tx.GetOne(diffBucket, base_encoding.Encode64ToBytes4(i))
+			diff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(i))
 			if err != nil {
 				return nil, err
 			}
@@ -549,7 +558,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 		}
 	} else {
 		for i := freshDumpSlot + clparams.SlotsPerDump; i > roundedSlot; i -= r.cfg.SlotsPerEpoch {
-			diff, err := tx.GetOne(diffBucket, base_encoding.Encode64ToBytes4(i))
+			diff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(i))
 			if err != nil {
 				return nil, err
 			}
@@ -573,7 +582,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 		return currentList, nil
 	}
 
-	slotDiff, err := tx.GetOne(diffBucket, base_encoding.Encode64ToBytes4(slot))
+	slotDiff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -626,8 +635,10 @@ func (r *HistoricalStatesReader) ReconstructUint64ListDump(tx kv.Tx, slot uint64
 }
 
 func (r *HistoricalStatesReader) ReadValidatorsForHistoricalState(tx kv.Tx, slot uint64) (*solid.ValidatorSet, error) {
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+
 	// Read the minimal beacon state which have the small fields.
-	sd, err := state_accessors.ReadSlotData(tx, slot)
+	sd, err := state_accessors.ReadSlotData(getter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -715,8 +726,9 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 	var beginSlot uint64
 	epoch, prevEpoch := r.computeRelevantEpochs(slot)
 	beginSlot = prevEpoch * r.cfg.SlotsPerEpoch
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
-	currentActiveIndicies, err := state_accessors.ReadActiveIndicies(tx, epoch*r.cfg.SlotsPerEpoch)
+	currentActiveIndicies, err := state_accessors.ReadActiveIndicies(getter, epoch*r.cfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -724,14 +736,14 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 	if epoch == 0 {
 		previousActiveIndicies = currentActiveIndicies
 	} else {
-		previousActiveIndicies, err = state_accessors.ReadActiveIndicies(tx, (epoch-1)*r.cfg.SlotsPerEpoch)
+		previousActiveIndicies, err = state_accessors.ReadActiveIndicies(getter, (epoch-1)*r.cfg.SlotsPerEpoch)
 		if err != nil {
 			return nil, nil, err
 		}
 	}
 
 	// Read the minimal beacon state which have the small fields.
-	sd, err := state_accessors.ReadSlotData(tx, slot)
+	sd, err := state_accessors.ReadSlotData(getter, slot)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -857,7 +869,8 @@ func (r *HistoricalStatesReader) tryCachingEpochsInParallell(tx kv.Tx, activeIdx
 }
 
 func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, slot uint64) (solid.Uint64ListSSZ, error) {
-	sd, err := state_accessors.ReadSlotData(tx, slot)
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+	sd, err := state_accessors.ReadSlotData(getter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -876,10 +889,11 @@ func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, slot uint64) (
 }
 
 func (r *HistoricalStatesReader) ReadRandaoMixBySlotAndIndex(tx kv.Tx, slot, index uint64) (common.Hash, error) {
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 	epoch := slot / r.cfg.SlotsPerEpoch
 	epochSubIndex := epoch % r.cfg.EpochsPerHistoricalVector
 	if index == epochSubIndex {
-		intraRandaoMix, err := tx.GetOne(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
+		intraRandaoMix, err := getter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
 		if err != nil {
 			return common.Hash{}, err
 		}
@@ -908,7 +922,7 @@ func (r *HistoricalStatesReader) ReadRandaoMixBySlotAndIndex(tx kv.Tx, slot, ind
 	if needFromGenesis {
 		return r.genesisState.GetRandaoMixes(epoch), nil
 	}
-	mixBytes, err := tx.GetOne(kv.RandaoMixes, base_encoding.Encode64ToBytes4(epochLookup*r.cfg.SlotsPerEpoch))
+	mixBytes, err := getter(kv.RandaoMixes, base_encoding.Encode64ToBytes4(epochLookup*r.cfg.SlotsPerEpoch))
 	if err != nil {
 		return common.Hash{}, err
 	}
diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 6e9fdf7b6e6..881e147ce91 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -18,16 +18,35 @@ package state_accessors
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
 	"github.com/erigontech/erigon/cl/cltypes/solid"
 	"github.com/erigontech/erigon/cl/persistence/base_encoding"
 	"github.com/erigontech/erigon/cl/phase1/core/state"
+	"github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 )
 
+type GetValFn func(table string, key []byte) ([]byte, error)
+
+func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
+	return func(table string, key []byte) ([]byte, error) {
+		if snapshot != nil {
+			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+			if err != nil {
+				return nil, err
+			}
+			if v != nil {
+				return v, nil
+			}
+		}
+		return tx.GetOne(table, key)
+	}
+}
+
 // InitializeValidatorTable initializes the validator table in the database.
 func InitializeStaticTables(tx kv.RwTx, state *state.CachingBeaconState) error {
 	var err error
@@ -164,9 +183,9 @@ func SetStateProcessingProgress(tx kv.RwTx, progress uint64) error {
 	return tx.Put(kv.StatesProcessingProgress, kv.StatesProcessingKey, base_encoding.Encode64ToBytes4(progress))
 }
 
-func ReadSlotData(tx kv.Tx, slot uint64) (*SlotData, error) {
+func ReadSlotData(getFn GetValFn, slot uint64) (*SlotData, error) {
 	sd := &SlotData{}
-	v, err := tx.GetOne(kv.SlotData, base_encoding.Encode64ToBytes4(slot))
+	v, err := getFn(kv.SlotData, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -178,9 +197,9 @@ func ReadSlotData(tx kv.Tx, slot uint64) (*SlotData, error) {
 	return sd, sd.ReadFrom(buf)
 }
 
-func ReadEpochData(tx kv.Tx, slot uint64) (*EpochData, error) {
+func ReadEpochData(getFn GetValFn, slot uint64) (*EpochData, error) {
 	ed := &EpochData{}
-	v, err := tx.GetOne(kv.EpochData, base_encoding.Encode64ToBytes4(slot))
+	v, err := getFn(kv.EpochData, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -193,10 +212,10 @@ func ReadEpochData(tx kv.Tx, slot uint64) (*EpochData, error) {
 }
 
 // ReadCheckpoints reads the checkpoints from the database, Current, Previous and Finalized
-func ReadCheckpoints(tx kv.Tx, slot uint64) (current solid.Checkpoint, previous solid.Checkpoint, finalized solid.Checkpoint, ok bool, err error) {
+func ReadCheckpoints(getFn GetValFn, slot uint64) (current solid.Checkpoint, previous solid.Checkpoint, finalized solid.Checkpoint, ok bool, err error) {
 	ed := &EpochData{}
 	var v []byte
-	v, err = tx.GetOne(kv.EpochData, base_encoding.Encode64ToBytes4(slot))
+	v, err = getFn(kv.EpochData, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return
 	}
@@ -212,8 +231,8 @@ func ReadCheckpoints(tx kv.Tx, slot uint64) (current solid.Checkpoint, previous
 }
 
 // ReadCheckpoints reads the checkpoints from the database, Current, Previous and Finalized
-func ReadNextSyncCommittee(tx kv.Tx, slot uint64) (committee *solid.SyncCommittee, err error) {
-	v, err := tx.GetOne(kv.NextSyncCommittee, base_encoding.Encode64ToBytes4(slot))
+func ReadNextSyncCommittee(getFn GetValFn, slot uint64) (committee *solid.SyncCommittee, err error) {
+	v, err := getFn(kv.NextSyncCommittee, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -226,8 +245,8 @@ func ReadNextSyncCommittee(tx kv.Tx, slot uint64) (committee *solid.SyncCommitte
 }
 
 // ReadCheckpoints reads the checkpoints from the database, Current, Previous and Finalized
-func ReadCurrentSyncCommittee(tx kv.Tx, slot uint64) (committee *solid.SyncCommittee, err error) {
-	v, err := tx.GetOne(kv.CurrentSyncCommittee, base_encoding.Encode64ToBytes4(slot))
+func ReadCurrentSyncCommittee(getFn GetValFn, slot uint64) (committee *solid.SyncCommittee, err error) {
+	v, err := getFn(kv.CurrentSyncCommittee, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -301,9 +320,9 @@ func ReadValidatorsTable(tx kv.Tx, out *StaticValidatorTable) error {
 	return err
 }
 
-func ReadActiveIndicies(tx kv.Tx, slot uint64) ([]uint64, error) {
+func ReadActiveIndicies(getFn GetValFn, slot uint64) ([]uint64, error) {
 	key := base_encoding.Encode64ToBytes4(slot)
-	v, err := tx.GetOne(kv.ActiveValidatorIndicies, key)
+	v, err := getFn(kv.ActiveValidatorIndicies, key)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go
index fa3a2a62f66..c8acac5da17 100644
--- a/cmd/caplin/caplin1/run.go
+++ b/cmd/caplin/caplin1/run.go
@@ -379,8 +379,8 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 			return err
 		}
 	}
-
-	antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, csn, rcsn, logger, states, backfilling, blobBackfilling, config.SnapshotGenerationEnabled, snBuildSema)
+	stateSnapshots := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, freezeblocks.MakeCaplinStateSnapshotsTypes(indexDB), logger)
+	antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, stateSnapshots, csn, rcsn, logger, states, backfilling, blobBackfilling, config.SnapshotGenerationEnabled, snBuildSema)
 	// Create the antiquary
 	go func() {
 		if err := antiq.Loop(); err != nil {
@@ -392,7 +392,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 		return err
 	}
 
-	statesReader := historical_states_reader.NewHistoricalStatesReader(beaconConfig, rcsn, vTables, genesisState)
+	statesReader := historical_states_reader.NewHistoricalStatesReader(beaconConfig, rcsn, vTables, genesisState, stateSnapshots)
 	validatorParameters := validator_params.NewValidatorParams()
 	if config.BeaconAPIRouter.Active {
 		apiHandler := handler.NewApiHandler(
@@ -427,6 +427,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 			proposerSlashingService,
 			option.builderClient,
 			validatorMonitor,
+			stateSnapshots,
 		)
 		go beacon.ListenAndServe(&beacon.LayeredBeaconHandler{
 			ArchiveApi: apiHandler,
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 813043149e9..1c32c032aa1 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -82,6 +82,8 @@ func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 			kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
 			kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
 			kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
+			kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
+			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
 	}
 }

From 0fd5197f4ac6f027040e63627933824da436a9f9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 18:34:21 +0200
Subject: [PATCH 040/131] save

---
 cmd/capcli/cli.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 5c1d0cf0f95..8a251989c92 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -1178,7 +1178,8 @@ func (c *DumpBlobsSnapshotsToStore) Run(ctx *Context) error {
 type DumpStateSnapshots struct {
 	chainCfg
 	outputFolder
-	To uint64 `name:"to" help:"slot to dump"`
+	To       uint64 `name:"to" help:"slot to dump"`
+	StepSize uint64 `name:"step-size" help:"step size" default:"10000"`
 }
 
 func (c *DumpStateSnapshots) Run(ctx *Context) error {
@@ -1217,7 +1218,7 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 		return err
 	}
 
-	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, 100_000, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
+	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, c.StepSize, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err
 	}
 	return nil

From f5c52bd60d7135355e47acec2a9c2319de0333c5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 18:38:19 +0200
Subject: [PATCH 041/131] save

---
 cl/antiquary/state_antiquary.go | 21 +++++++++++++++++++++
 cmd/caplin/caplin1/run.go       |  3 +++
 2 files changed, 24 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 92684897495..d8327bb0c42 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
@@ -27,6 +28,7 @@ import (
 
 	"github.com/erigontech/erigon-lib/common"
 	libcommon "github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/downloader/snaptype"
 	"github.com/erigontech/erigon-lib/etl"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/log/v3"
@@ -413,6 +415,25 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		return err
 	}
 	log.Info("Historical states antiquated", "slot", s.currentState.Slot(), "root", libcommon.Hash(stateRoot), "latency", endTime)
+	if s.snapgen {
+		if err := s.stateSn.OpenFolder(); err != nil {
+			return err
+		}
+		if err := s.stateSn.DumpCaplinState(
+			ctx,
+			s.stateSn.BlocksAvailable()+1,
+			s.currentState.Slot(),
+			snaptype.CaplinMergeLimit,
+			s.sn.Salt,
+			s.dirs,
+			runtime.NumCPU(),
+			log.LvlDebug,
+			s.logger,
+		); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
diff --git a/cmd/caplin/caplin1/run.go b/cmd/caplin/caplin1/run.go
index c8acac5da17..46063885805 100644
--- a/cmd/caplin/caplin1/run.go
+++ b/cmd/caplin/caplin1/run.go
@@ -380,6 +380,9 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 		}
 	}
 	stateSnapshots := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, freezeblocks.MakeCaplinStateSnapshotsTypes(indexDB), logger)
+	if err := stateSnapshots.OpenFolder(); err != nil {
+		return err
+	}
 	antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, stateSnapshots, csn, rcsn, logger, states, backfilling, blobBackfilling, config.SnapshotGenerationEnabled, snBuildSema)
 	// Create the antiquary
 	go func() {

From 35f6c3d06885649a1fa99d88a10ee259e90ce44a Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 19:28:39 +0200
Subject: [PATCH 042/131] save

---
 .../historical_states_reader.go               | 62 +++++++++++++++----
 1 file changed, 51 insertions(+), 11 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 635390e9daa..5541a4f2cbf 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -289,23 +289,30 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 	}
 
 	needFromDB := size - needFromGenesis
-	cursor, err := tx.Cursor(table)
+	highestAvaiableSlot, err := r.highestSlotInSnapshotsAndDB(tx, table)
 	if err != nil {
 		return err
 	}
-	defer cursor.Close()
+
 	var currKeySlot uint64
-	for k, v, err := cursor.Seek(base_encoding.Encode64ToBytes4(slot - needFromDB)); err == nil && k != nil; k, v, err = cursor.Next() {
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+	for i := slot - needFromDB; i <= highestAvaiableSlot; i++ {
+		key := base_encoding.Encode64ToBytes4(i)
+		v, err := getter(table, key)
+		if err != nil {
+			return err
+		}
 		if len(v) != 32 {
-			return fmt.Errorf("invalid key %x", k)
+			return fmt.Errorf("invalid key %x", key)
 		}
-		currKeySlot = base_encoding.Decode64FromBytes4(k)
+		currKeySlot = i
 		out.Set(int(currKeySlot%size), common.BytesToHash(v))
 		inserted++
 		if inserted == needFromDB {
 			break
 		}
 	}
+
 	for i := 0; i < int(needFromGenesis); i++ {
 		currKeySlot++
 		out.Set(int(currKeySlot%size), genesisVector.Get(int(currKeySlot%size)))
@@ -351,6 +358,30 @@ func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength
 	return nil
 }
 
+func (r *HistoricalStatesReader) highestSlotInSnapshotsAndDB(tx kv.Tx, tbl string) (uint64, error) {
+	cursor, err := tx.Cursor(tbl)
+	if err != nil {
+		return 0, err
+	}
+	defer cursor.Close()
+	k, _, err := cursor.Last()
+	if err != nil {
+		return 0, err
+	}
+	if k == nil {
+		if r.stateSn != nil {
+			return r.stateSn.BlocksAvailable(), nil
+		}
+		return 0, nil
+	}
+	avaiableInDB := base_encoding.Decode64FromBytes4(k)
+	var availableInSnapshots uint64
+	if r.stateSn != nil {
+		availableInSnapshots = r.stateSn.BlocksAvailable()
+	}
+	return max(avaiableInDB, availableInSnapshots), nil
+}
+
 func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out solid.HashVectorSSZ) error {
 	size := r.cfg.EpochsPerHistoricalVector
 	genesisVector := r.genesisState.RandaoMixes()
@@ -361,19 +392,29 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 	if size > epoch || epoch-size <= genesisEpoch {
 		needFromGenesis = size - (epoch - genesisEpoch)
 	}
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	needFromDB := size - needFromGenesis
-	cursor, err := tx.Cursor(kv.RandaoMixes)
+
+	highestAvaiableSlot, err := r.highestSlotInSnapshotsAndDB(tx, kv.RandaoMixes)
 	if err != nil {
 		return err
 	}
-	defer cursor.Close()
 	var currKeyEpoch uint64
-	for k, v, err := cursor.Seek(base_encoding.Encode64ToBytes4(roundedSlot - (needFromDB)*r.cfg.SlotsPerEpoch)); err == nil && k != nil; k, v, err = cursor.Next() {
+
+	for i := roundedSlot - (needFromDB)*r.cfg.SlotsPerEpoch; i <= highestAvaiableSlot; i++ {
+		key := base_encoding.Encode64ToBytes4(i)
+		v, err := getter(kv.RandaoMixes, key)
+		if err != nil {
+			return err
+		}
+		if len(v) == 0 {
+			continue
+		}
 		if len(v) != 32 {
-			return fmt.Errorf("invalid key %x", k)
+			return fmt.Errorf("invalid key %x", key)
 		}
-		currKeyEpoch = base_encoding.Decode64FromBytes4(k) / r.cfg.SlotsPerEpoch
+		currKeyEpoch = i / r.cfg.SlotsPerEpoch
 		out.Set(int(currKeyEpoch%size), common.BytesToHash(v))
 		inserted++
 		if inserted == needFromDB {
@@ -384,7 +425,6 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 		currKeyEpoch++
 		out.Set(int(currKeyEpoch%size), genesisVector.Get(int(currKeyEpoch%size)))
 	}
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	// Now we need to read the intra epoch randao mix.
 	intraRandaoMix, err := getter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))

From 8d714047a00128e34d422ac8ff3582f599ed2ece Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 20:08:39 +0200
Subject: [PATCH 043/131] save

---
 .../historical_states_reader.go               | 127 +++++++++++-------
 1 file changed, 76 insertions(+), 51 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 5541a4f2cbf..d87daa8e637 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -322,16 +322,6 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 
 func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength, slot uint64, out *solid.ListSSZ[*cltypes.Eth1Data]) error {
 	initialSlot := r.cfg.RoundSlotToVotePeriod(slot)
-	initialKey := base_encoding.Encode64ToBytes4(initialSlot)
-	cursor, err := tx.Cursor(kv.Eth1DataVotes)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-	k, v, err := cursor.Seek(initialKey)
-	if err != nil {
-		return err
-	}
 	if initialSlot <= r.genesisState.Slot() {
 		// We need to prepend the genesis votes
 		for i := 0; i < r.genesisState.Eth1DataVotes().Len(); i++ {
@@ -339,22 +329,28 @@ func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength
 		}
 	}
 
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 	endSlot := r.cfg.RoundSlotToVotePeriod(slot + r.cfg.SlotsPerEpoch*r.cfg.EpochsPerEth1VotingPeriod)
 
-	for k != nil && base_encoding.Decode64FromBytes4(k) < endSlot {
+	for i := initialSlot; i < endSlot; i++ {
 		if out.Len() >= int(eth1DataVotesLength) {
 			break
 		}
+		key := base_encoding.Encode64ToBytes4(i)
+		v, err := getter(kv.Eth1DataVotes, key)
+		if err != nil {
+			return err
+		}
+		if len(v) == 0 {
+			continue
+		}
 		eth1Data := &cltypes.Eth1Data{}
 		if err := eth1Data.DecodeSSZ(v, 0); err != nil {
 			return err
 		}
 		out.Append(eth1Data)
-		k, v, err = cursor.Next()
-		if err != nil {
-			return err
-		}
 	}
+
 	return nil
 }
 
@@ -486,43 +482,79 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 		return nil, err
 	}
 
-	diffCursor, err := tx.Cursor(diffBucket)
+	highestSlotAvailable, err := r.highestSlotInSnapshotsAndDB(tx, diffBucket)
 	if err != nil {
 		return nil, err
 	}
-	defer diffCursor.Close()
 	if forward {
-		for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= slot; k, v, err = diffCursor.Next() {
+		// for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= slot; k, v, err = diffCursor.Next() {
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	if len(k) != 4 {
+		// 		return nil, fmt.Errorf("invalid key %x", k)
+		// 	}
+		// 	currSlot := base_encoding.Decode64FromBytes4(k)
+		// 	if currSlot == freshDumpSlot {
+		// 		continue
+		// 	}
+		// 	if currSlot > slot {
+		// 		return nil, fmt.Errorf("diff not found for slot %d", slot)
+		// 	}
+		// 	currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, false)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// }
+		for currSlot := freshDumpSlot; currSlot <= slot && currSlot < highestSlotAvailable; currSlot++ {
+			key := base_encoding.Encode64ToBytes4(currSlot)
+			v, err := getter(diffBucket, key)
 			if err != nil {
 				return nil, err
 			}
-			if len(k) != 4 {
-				return nil, fmt.Errorf("invalid key %x", k)
+			if len(v) == 0 {
+				continue
+			}
+			if len(key) != 4 {
+				return nil, fmt.Errorf("invalid key %x", key)
 			}
-			currSlot := base_encoding.Decode64FromBytes4(k)
 			if currSlot == freshDumpSlot {
 				continue
 			}
-			if currSlot > slot {
-				return nil, fmt.Errorf("diff not found for slot %d", slot)
-			}
 			currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, false)
 			if err != nil {
 				return nil, err
 			}
 		}
 	} else {
-		for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot + clparams.SlotsPerDump)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) > slot; k, v, err = diffCursor.Prev() {
+		// for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot + clparams.SlotsPerDump)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) > slot; k, v, err = diffCursor.Prev() {
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	if len(k) != 4 {
+		// 		return nil, fmt.Errorf("invalid key %x", k)
+		// 	}
+		// 	currSlot := base_encoding.Decode64FromBytes4(k)
+		// 	if currSlot <= slot || currSlot > freshDumpSlot+clparams.SlotsPerDump {
+		// 		continue
+		// 	}
+		// 	currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, true)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// }
+		for currSlot := freshDumpSlot + clparams.SlotsPerDump; currSlot > slot && currSlot > r.genesisState.Slot(); currSlot-- {
+			key := base_encoding.Encode64ToBytes4(currSlot)
+			v, err := getter(diffBucket, key)
 			if err != nil {
 				return nil, err
 			}
-			if len(k) != 4 {
-				return nil, fmt.Errorf("invalid key %x", k)
-			}
-			currSlot := base_encoding.Decode64FromBytes4(k)
-			if currSlot <= slot || currSlot > freshDumpSlot+clparams.SlotsPerDump {
+			if len(v) == 0 {
 				continue
 			}
+			if len(key) != 4 {
+				return nil, fmt.Errorf("invalid key %x", key)
+			}
 			currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, true)
 			if err != nil {
 				return nil, err
@@ -612,11 +644,6 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 		}
 	}
 
-	diffCursor, err := tx.Cursor(diffBucket)
-	if err != nil {
-		return nil, err
-	}
-	defer diffCursor.Close()
 	if slot%r.cfg.SlotsPerEpoch == 0 {
 		currentList = currentList[:validatorSetLength*8]
 		return currentList, nil
@@ -635,26 +662,24 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 }
 
 func (r *HistoricalStatesReader) ReconstructUint64ListDump(tx kv.Tx, slot uint64, bkt string, size int, out solid.Uint64ListSSZ) error {
-	diffCursor, err := tx.Cursor(bkt)
-	if err != nil {
-		return err
-	}
-	defer diffCursor.Close()
-
-	k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(slot))
-	if err != nil {
-		return err
-	}
-	if k == nil {
-		return fmt.Errorf("diff not found for slot %d", slot)
-	}
-	keySlot := base_encoding.Decode64FromBytes4(k)
-	if keySlot > slot {
-		_, v, err = diffCursor.Prev()
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+	var (
+		v   []byte
+		err error
+	)
+	// Try seeking <= to slot
+	for i := slot; i >= r.genesisState.Slot(); i-- {
+		key := base_encoding.Encode64ToBytes4(i)
+		v, err = getter(bkt, key)
 		if err != nil {
 			return err
 		}
+		if len(v) == 0 {
+			continue
+		}
+		break
 	}
+
 	var b bytes.Buffer
 	if _, err := b.Write(v); err != nil {
 		return err

From 296e2e517f27dd1d236baebb4d0cb36cfa3365f7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 21:06:51 +0200
Subject: [PATCH 044/131] save

---
 .../freezeblocks/caplin_state_snapshots.go    | 28 +++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 1c32c032aa1..2a7d408719a 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -65,7 +65,7 @@ func getKvGetterForStateTable(db kv.RoDB, tableName string) KeyValueGetter {
 
 func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 	return SnapshotTypes{
-		Types: map[string]KeyValueGetter{
+		KeyValueGetters: map[string]KeyValueGetter{
 			kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
 			kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
 			kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
@@ -85,6 +85,11 @@ func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 			kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
 			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
+		Compression: map[string]bool{
+			kv.EpochData:   true,
+			kv.SlotData:    true,
+			kv.StateEvents: true,
+		},
 	}
 }
 
@@ -120,7 +125,8 @@ type CaplinStateSnapshots struct {
 type KeyValueGetter func(numId uint64) ([]byte, []byte, error)
 
 type SnapshotTypes struct {
-	Types map[string]KeyValueGetter
+	KeyValueGetters map[string]KeyValueGetter
+	Compression     map[string]bool
 }
 
 // NewCaplinStateSnapshots - opens all snapshots. But to simplify everything:
@@ -136,7 +142,7 @@ func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.B
 	// 	DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 	// }
 	Segments := make(map[string]*segments)
-	for k := range snapshotTypes.Types {
+	for k := range snapshotTypes.KeyValueGetters {
 		Segments[strings.ToLower(k)] = &segments{
 			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 		}
@@ -505,7 +511,7 @@ func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegme
 	return nil, false
 }
 
-func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
+func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGetter, fromSlot uint64, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger, compress bool) error {
 	tmpDir, snapDir := dirs.Tmp, dirs.SnapCaplin
 
 	segName := snaptype.BeaconBlocks.FileName(0, fromSlot, toSlot)
@@ -531,8 +537,14 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		if i%20_000 == 0 {
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
 		}
-		if err := sn.AddUncompressedWord(dump); err != nil {
-			return err
+		if compress {
+			if err := sn.AddWord(dump); err != nil {
+				return err
+			}
+		} else {
+			if err := sn.AddUncompressedWord(dump); err != nil {
+				return err
+			}
 		}
 	}
 	if sn.Count() != int(blocksPerFile) {
@@ -581,7 +593,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 	fromSlot = (fromSlot / blocksPerFile) * blocksPerFile
 	toSlot = (toSlot / blocksPerFile) * blocksPerFile
 	fmt.Println("DumpCaplinState", fromSlot, toSlot)
-	for snapName, kvGetter := range s.snapshotTypes.Types {
+	for snapName, kvGetter := range s.snapshotTypes.KeyValueGetters {
 		for i := fromSlot; i < toSlot; i += blocksPerFile {
 			if toSlot-i < blocksPerFile {
 				break
@@ -589,7 +601,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 			// keep beaconblocks here but whatever....
 			to := i + blocksPerFile
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
-			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, blocksPerFile, salt, dirs, workers, lvl, logger); err != nil {
+			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, blocksPerFile, salt, dirs, workers, lvl, logger, s.snapshotTypes.Compression[snapName]); err != nil {
 				return err
 			}
 		}

From 9efb11256d48e30a7015a58298b53ca3d536aefc Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 21:07:38 +0200
Subject: [PATCH 045/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 2a7d408719a..602a89d5f2a 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -592,8 +592,8 @@ func simpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir st
 func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, toSlot, blocksPerFile uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
 	fromSlot = (fromSlot / blocksPerFile) * blocksPerFile
 	toSlot = (toSlot / blocksPerFile) * blocksPerFile
-	fmt.Println("DumpCaplinState", fromSlot, toSlot)
 	for snapName, kvGetter := range s.snapshotTypes.KeyValueGetters {
+		snapName = strings.ToLower(snapName)
 		for i := fromSlot; i < toSlot; i += blocksPerFile {
 			if toSlot-i < blocksPerFile {
 				break

From 1421683b359efe2205f6a644942eadb4ef2c1d4a Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 21:35:25 +0200
Subject: [PATCH 046/131] save

---
 cmd/capcli/cli.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 8a251989c92..7add584a4a7 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -180,7 +180,7 @@ func (c *Chain) Run(ctx *Context) error {
 	}
 
 	downloader := network.NewBackwardBeaconDownloader(ctx, beacon, nil, nil, db)
-	cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, true, false, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root())
+	cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, true, false, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root())
 	return stages.SpawnStageHistoryDownload(cfg, ctx, log.Root())
 }
 
@@ -582,7 +582,10 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 		return err
 	}
 
-	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot)
+	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
+	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
+
+	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
 	start := time.Now()
 	haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)
 	if err != nil {

From 0a79e0edac9de355c19ce4f5212487fd9ab2320f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 21:39:57 +0200
Subject: [PATCH 047/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 602a89d5f2a..375ab7ca795 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -86,9 +86,10 @@ func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
 		Compression: map[string]bool{
-			kv.EpochData:   true,
-			kv.SlotData:    true,
-			kv.StateEvents: true,
+			kv.EpochData:     true,
+			kv.SlotData:      true,
+			kv.StateEvents:   true,
+			kv.Eth1DataVotes: true,
 		},
 	}
 }
@@ -593,7 +594,6 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 	fromSlot = (fromSlot / blocksPerFile) * blocksPerFile
 	toSlot = (toSlot / blocksPerFile) * blocksPerFile
 	for snapName, kvGetter := range s.snapshotTypes.KeyValueGetters {
-		snapName = strings.ToLower(snapName)
 		for i := fromSlot; i < toSlot; i += blocksPerFile {
 			if toSlot-i < blocksPerFile {
 				break

From d26fd6cf6ef6ef856fdaa5471328cd4a8bc169d6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 21:51:30 +0200
Subject: [PATCH 048/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 375ab7ca795..83851c2c70f 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -85,12 +85,7 @@ func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 			kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
 			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
-		Compression: map[string]bool{
-			kv.EpochData:     true,
-			kv.SlotData:      true,
-			kv.StateEvents:   true,
-			kv.Eth1DataVotes: true,
-		},
+		Compression: map[string]bool{},
 	}
 }
 

From 9ba345770739034ec3cca28da9e5324f4a0946e7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:25:53 +0200
Subject: [PATCH 049/131] save

---
 cl/persistence/state/state_accessors.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 881e147ce91..3ecd54a9441 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -19,6 +19,7 @@ package state_accessors
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -40,6 +41,7 @@ func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots
 				return nil, err
 			}
 			if v != nil {
+				fmt.Println("hit")
 				return v, nil
 			}
 		}

From 9a69e31012d9e915d5a581638dda50531b0af37b Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:41:51 +0200
Subject: [PATCH 050/131] save

---
 cl/persistence/state/state_accessors.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 3ecd54a9441..8f6ea169927 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -40,6 +40,7 @@ func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots
 			if err != nil {
 				return nil, err
 			}
+			fmt.Println(v)
 			if v != nil {
 				fmt.Println("hit")
 				return v, nil

From 7462a5caf30a04a00f206b630b43909fdc4be8b3 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:43:11 +0200
Subject: [PATCH 051/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 83851c2c70f..232fa1af086 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -654,6 +654,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
+		fmt.Println("not ok")
 		return nil, nil
 	}
 

From a23be29dc0947ae38901ade7089c8ff798ec69ba Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:44:35 +0200
Subject: [PATCH 052/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 232fa1af086..2773bc5b142 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -654,7 +654,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
-		fmt.Println("not ok")
+		fmt.Println("not ok", view.roTxs)
 		return nil, nil
 	}
 

From ede1c4866bbc038e9fb3d680f4d6f737e420d3dc Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:45:13 +0200
Subject: [PATCH 053/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 2773bc5b142..231e74873a1 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -654,7 +654,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
-		fmt.Println("not ok", view.roTxs)
+		fmt.Println("not ok", strings.ToLower(tbl), iew.roTxs)
 		return nil, nil
 	}
 

From 44fc37f58c95228fb9edd5cd5e7f6b010d9cfb79 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:45:30 +0200
Subject: [PATCH 054/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 231e74873a1..689b29e59d5 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -654,7 +654,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
-		fmt.Println("not ok", strings.ToLower(tbl), iew.roTxs)
+		fmt.Println("not ok", strings.ToLower(tbl), view.roTxs)
 		return nil, nil
 	}
 

From 9ec82fd165b4e1b8d5987e8d06841e7cc76938b0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:47:18 +0200
Subject: [PATCH 055/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 689b29e59d5..61f1b9ac203 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -494,10 +494,12 @@ func (v *CaplinStateView) Close() {
 }
 
 func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
+	fmt.Println(v.s.Segments[tbl].VisibleSegments, tbl)
 	return v.s.Segments[tbl].VisibleSegments
 }
 
 func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {
+	f
 	for _, seg := range v.VisibleSegments(tbl) {
 		if !(slot >= seg.from && slot < seg.to) {
 			continue
@@ -654,7 +656,6 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
-		fmt.Println("not ok", strings.ToLower(tbl), view.roTxs)
 		return nil, nil
 	}
 

From 17db8c949b518ade27db966b27423b0085879334 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:47:24 +0200
Subject: [PATCH 056/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 61f1b9ac203..681a1585f8a 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -499,7 +499,6 @@ func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
 }
 
 func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {
-	f
 	for _, seg := range v.VisibleSegments(tbl) {
 		if !(slot >= seg.from && slot < seg.to) {
 			continue

From a43d9d5080cd4a4bc19ff7dc9fc72ef5ba15bbfc Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:48:35 +0200
Subject: [PATCH 057/131] save

---
 cmd/capcli/cli.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 7add584a4a7..cd19a52d62a 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -584,7 +584,9 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 
 	snTypes := freezeblocks.MakeCaplinStateSnapshotsTypes(db)
 	stateSn := freezeblocks.NewCaplinStateSnapshots(ethconfig.BlocksFreezing{}, beaconConfig, dirs, snTypes, log.Root())
-
+	if err := stateSn.OpenFolder(); err != nil {
+		return err
+	}
 	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
 	start := time.Now()
 	haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)

From f94332ddc46906b73ed08e40debff2f282bbfaa8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 22:49:59 +0200
Subject: [PATCH 058/131] save

---
 cl/persistence/state/state_accessors.go                   | 3 ---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 8f6ea169927..881e147ce91 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -19,7 +19,6 @@ package state_accessors
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -40,9 +39,7 @@ func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots
 			if err != nil {
 				return nil, err
 			}
-			fmt.Println(v)
 			if v != nil {
-				fmt.Println("hit")
 				return v, nil
 			}
 		}
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 681a1585f8a..c5c6a13ce7e 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -262,7 +262,6 @@ Loop:
 		}
 		filePath := filepath.Join(s.dir, fName)
 		if err := s.openSegIfNeed(sn, filePath); err != nil {
-			fmt.Println(err)
 			if errors.Is(err, os.ErrNotExist) {
 				if optimistic {
 					continue Loop
@@ -284,7 +283,6 @@ Loop:
 			segments.DirtySegments.Set(sn)
 		}
 		if err := openIdxForCaplinStateIfNeeded(sn, filePath, optimistic); err != nil {
-			fmt.Println(err)
 			return err
 		}
 		// Only bob sidecars count for progression
@@ -494,7 +492,6 @@ func (v *CaplinStateView) Close() {
 }
 
 func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
-	fmt.Println(v.s.Segments[tbl].VisibleSegments, tbl)
 	return v.s.Segments[tbl].VisibleSegments
 }
 

From 6370a83c25dbed2c3d5728a5c6fef8b78d4ad6a0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:06:44 +0200
Subject: [PATCH 059/131] save

---
 cl/phase1/core/state/raw/hashing.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index a4390593baf..79a286cf01f 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,6 +17,7 @@
 package raw
 
 import (
+	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -31,9 +32,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	// for i := 0; i < len(b.leaves); i += 32 {
-	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	// }
+	for i := 0; i < len(b.leaves); i += 32 {
+		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	}
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return

From 6c4da9ffd26e996e7de22e8c9605b86edbe2b2d2 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:09:24 +0200
Subject: [PATCH 060/131] save

---
 cl/persistence/state/state_accessors.go | 19 +++++++--------
 cl/phase1/core/state/raw/hashing.go     |  7 +++---
 holy.txt                                | 32 +++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 14 deletions(-)
 create mode 100644 holy.txt

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 881e147ce91..40cc66b9489 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -18,7 +18,6 @@ package state_accessors
 
 import (
 	"bytes"
-	"encoding/binary"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -34,15 +33,15 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
-		if snapshot != nil {
-			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
-			if err != nil {
-				return nil, err
-			}
-			if v != nil {
-				return v, nil
-			}
-		}
+		// if snapshot != nil {
+		// 	v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	if v != nil {
+		// 		return v, nil
+		// 	}
+		// }
 		return tx.GetOne(table, key)
 	}
 }
diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index 79a286cf01f..a4390593baf 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,7 +17,6 @@
 package raw
 
 import (
-	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -32,9 +31,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	for i := 0; i < len(b.leaves); i += 32 {
-		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	}
+	// for i := 0; i < len(b.leaves); i += 32 {
+	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	// }
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return
diff --git a/holy.txt b/holy.txt
new file mode 100644
index 00000000000..921062010cf
--- /dev/null
+++ b/holy.txt
@@ -0,0 +1,32 @@
+0 0x607db06200000000000000000000000000000000000000000000000000000000                                                                                                                                                  
+1 0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078                                                                                                                                                  
+2 0x41420f0000000000000000000000000000000000000000000000000000000000                                                                                                                                                  
+3 0x30680b18ada5ceaeb222d0d15bd7058fbc4e1e5e3c002d504adee82cc013a517                                                                                                                                                  
+4 0x1158a7769cb0dfc3d015aa03bd76760d57bab8ad2fb77885a07917f1754194cf                                                                                                                                                  
+5 0xfe1acfde1678da01248a9825ec08d4f3ee73615c1263095077ee23063d1819cb                                                                                                                                                  
+6 0x0ead07fe6e6026ee73f966aceffaaeff6b50d6df923e5230a9cc33023d52adbe                                                                                                                                                  
+7 0xb6049c0ff9220d02083c30474400cbcb358a6d6804082757553468badccd8a48                                                                                                                                                  
+8 0xa4007911445daf00197d2cca907cebf0529773561b702a2136decb52edc87348                                                                                                                                                  
+9 0xc231c219f35945a38dbab773bf4f32c9080c59f9e6284fbb92ffe02c3318165d
+10 0x9301000000000000000000000000000000000000000000000000000000000000
+11 0xed774cba995d2b5cd55a05ac5126d4fb24a1dbf7523ebdc0b1dc8f7f02de30d6
+12 0xe7c27b3e6aaccb0c4f42eda4ffadd5f92fc28edfa5ef573f28ab5495f669282a
+13 0xe65a9db1e18c1435832c978ca666c5502dab9b85bf1258623e8a2e96e986e78f
+14 0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220
+15 0xe15e05e8cb173b5d2a4bc5acea5aab50ea2e616fcc41bee9833377ce40135897
+16 0xe9eddec5525f068001fdff036039e7946248a1d80ce8f6b40a1e79d01303519b
+17 0x0f00000000000000000000000000000000000000000000000000000000000000
+18 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
+19 0xe74aa97c38fea5a58af5514cd78e2180f0ba62f0709676c62d6e9af62fff77d0
+20 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
+21 0x3d4be5d019ba15ea3ef304a83b8a067f2e79f46a3fac8069306a6c814a0a35eb
+22 0x2961343e46fbbea5ef5373a5db989b5e2ed19baef61d9f26ee6ac63a9023862b
+23 0x3843792806b2d08dd125002fd3e68d117c7860e4c863f7f60d59625a6f220d48
+24 0x69c41e47f10b14f920a3851f8a4f71d19f83a8d1e28204efccccd644747141be
+25 0x0000000000000000000000000000000000000000000000000000000000000000
+26 0x0000000000000000000000000000000000000000000000000000000000000000
+27 0x0000000000000000000000000000000000000000000000000000000000000000
+28 0x0000000000000000000000000000000000000000000000000000000000000000
+29 0x0000000000000000000000000000000000000000000000000000000000000000
+30 0x0000000000000000000000000000000000000000000000000000000000000000
+31 0x0000000000000000000000000000000000000000000000000000000000000000

From 5ed3f7279b3a23c0d85786de18cfd0eb08d9b91d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:10:28 +0200
Subject: [PATCH 061/131] save

---
 cl/phase1/core/state/raw/hashing.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index a4390593baf..79a286cf01f 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,6 +17,7 @@
 package raw
 
 import (
+	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -31,9 +32,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	// for i := 0; i < len(b.leaves); i += 32 {
-	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	// }
+	for i := 0; i < len(b.leaves); i += 32 {
+		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	}
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return

From c9c69d16c0b37063a550ebed3b18b4c96e658c5d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:20:38 +0200
Subject: [PATCH 062/131] save

---
 .../state/historical_states_reader/historical_states_reader.go   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index d87daa8e637..41b59dfd454 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -308,6 +308,7 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 		currKeySlot = i
 		out.Set(int(currKeySlot%size), common.BytesToHash(v))
 		inserted++
+		fmt.Println(i, common.BytesToHash(v))
 		if inserted == needFromDB {
 			break
 		}

From 155b5f27758807c9b2bb90614c1bbeb8bd88e3d4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:20:52 +0200
Subject: [PATCH 063/131] save

---
 cl/persistence/state/state_accessors.go | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 40cc66b9489..881e147ce91 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -18,6 +18,7 @@ package state_accessors
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -33,15 +34,15 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
-		// if snapshot != nil {
-		// 	v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// 	if v != nil {
-		// 		return v, nil
-		// 	}
-		// }
+		if snapshot != nil {
+			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+			if err != nil {
+				return nil, err
+			}
+			if v != nil {
+				return v, nil
+			}
+		}
 		return tx.GetOne(table, key)
 	}
 }

From 9a4a46e50cbfe5f7c5fe1520af49f5a218916976 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:23:03 +0200
Subject: [PATCH 064/131] save

---
 .../historical_states_reader/historical_states_reader.go      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 41b59dfd454..c2ded11e5b6 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -308,7 +308,9 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 		currKeySlot = i
 		out.Set(int(currKeySlot%size), common.BytesToHash(v))
 		inserted++
-		fmt.Println(i, common.BytesToHash(v))
+		if table == kv.BlockRoot {
+			fmt.Println(i, common.BytesToHash(v))
+		}
 		if inserted == needFromDB {
 			break
 		}

From 39b4336553030697f733b04a22bb649fc27e4e19 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:25:02 +0200
Subject: [PATCH 065/131] save

---
 cl/persistence/state/state_accessors.go | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 881e147ce91..40cc66b9489 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -18,7 +18,6 @@ package state_accessors
 
 import (
 	"bytes"
-	"encoding/binary"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -34,15 +33,15 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
-		if snapshot != nil {
-			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
-			if err != nil {
-				return nil, err
-			}
-			if v != nil {
-				return v, nil
-			}
-		}
+		// if snapshot != nil {
+		// 	v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	if v != nil {
+		// 		return v, nil
+		// 	}
+		// }
 		return tx.GetOne(table, key)
 	}
 }

From 557484475eca81ace8de68540daf759abd11b09c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:25:59 +0200
Subject: [PATCH 066/131] save

---
 cmd/capcli/cli.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index cd19a52d62a..4574b4f1376 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -643,11 +643,11 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 		return err
 	}
 	if hRoot != wRoot {
-		// for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
-		// 	if haveState.PreviousEpochParticipation().Get(i) != wantState.PreviousEpochParticipation().Get(i) {
-		// 		log.Info("Participation mismatch", "index", i, "have", haveState.PreviousEpochParticipation().Get(i), "want", wantState.PreviousEpochParticipation().Get(i))
-		// 	}
-		// }
+		for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
+			if haveState.BlockRoots().Get(i) != wantState.BlockRoots().Get(i) {
+				log.Info("block roots mismatch", "index", i, "have", haveState.BlockRoots().Get(i), "want", wantState.BlockRoots().Get(i))
+			}
+		}
 		return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot))
 	}
 	return nil

From 33dab6319eb783f1138899df5a408d25ec237fb6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:33:19 +0200
Subject: [PATCH 067/131] save

---
 cl/persistence/state/state_accessors.go | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 40cc66b9489..881e147ce91 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -18,6 +18,7 @@ package state_accessors
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -33,15 +34,15 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
-		// if snapshot != nil {
-		// 	v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// 	if v != nil {
-		// 		return v, nil
-		// 	}
-		// }
+		if snapshot != nil {
+			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+			if err != nil {
+				return nil, err
+			}
+			if v != nil {
+				return v, nil
+			}
+		}
 		return tx.GetOne(table, key)
 	}
 }

From 1c208beab77951c28e9eec7c6c7a78e7913858d5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sat, 26 Oct 2024 23:37:41 +0200
Subject: [PATCH 068/131] save

---
 .../historical_states_reader/historical_states_reader.go      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index c2ded11e5b6..193c8c502b3 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -309,7 +309,9 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 		out.Set(int(currKeySlot%size), common.BytesToHash(v))
 		inserted++
 		if table == kv.BlockRoot {
-			fmt.Println(i, common.BytesToHash(v))
+			if int(currKeySlot%size) == 0 {
+				fmt.Println(i, common.BytesToHash(v))
+			}
 		}
 		if inserted == needFromDB {
 			break

From 66b0d8d6b6c958ffe6a1c752059e002a3de5705d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 00:47:24 +0200
Subject: [PATCH 069/131] save

---
 turbo/snapshotsync/freezeblocks/block_snapshots.go    |  3 +++
 .../freezeblocks/caplin_state_snapshots.go            | 11 ++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 0593d8453e7..7c8736e99b9 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -223,6 +223,9 @@ type DirtySegment struct {
 	refcount atomic.Int32
 
 	canDelete atomic.Bool
+
+	// caplin state snapshots only
+	filePath string
 }
 
 type VisibleSegment struct {
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index c5c6a13ce7e..579f5ed1f47 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -239,12 +239,13 @@ Loop:
 		if !ok {
 			continue
 		}
+		filePath := filepath.Join(s.dir, fName)
 		segments.DirtySegments.Walk(func(segments []*DirtySegment) bool {
 			for _, sn2 := range segments {
 				if sn2.Decompressor == nil { // it's ok if some segment was not able to open
 					continue
 				}
-				if fName == sn2.FileName() {
+				if filePath == sn2.filePath {
 					sn = sn2
 					exists = true
 					break
@@ -255,12 +256,12 @@ Loop:
 		if !exists {
 			sn = &DirtySegment{
 				// segType: f.Type, Unsupported
-				version: f.Version,
-				Range:   Range{f.From, f.To},
-				frozen:  snapcfg.IsFrozen(s.cfg.ChainName, f),
+				version:  f.Version,
+				Range:    Range{f.From, f.To},
+				frozen:   snapcfg.IsFrozen(s.cfg.ChainName, f),
+				filePath: filePath,
 			}
 		}
-		filePath := filepath.Join(s.dir, fName)
 		if err := s.openSegIfNeed(sn, filePath); err != nil {
 			if errors.Is(err, os.ErrNotExist) {
 				if optimistic {

From ced12474c6a5a6df9b848e4bc417e9f3e52844fc Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 00:56:09 +0200
Subject: [PATCH 070/131] save

---
 .../freezeblocks/caplin_state_snapshots.go    | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 579f5ed1f47..b8c32807d52 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -66,24 +66,24 @@ func getKvGetterForStateTable(db kv.RoDB, tableName string) KeyValueGetter {
 func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 	return SnapshotTypes{
 		KeyValueGetters: map[string]KeyValueGetter{
-			kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
-			kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
-			kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
-			kv.StateEvents:               getKvGetterForStateTable(db, kv.StateEvents),
-			kv.ActiveValidatorIndicies:   getKvGetterForStateTable(db, kv.ActiveValidatorIndicies),
-			kv.StateRoot:                 getKvGetterForStateTable(db, kv.StateRoot),
-			kv.BlockRoot:                 getKvGetterForStateTable(db, kv.BlockRoot),
-			kv.SlotData:                  getKvGetterForStateTable(db, kv.SlotData),
-			kv.EpochData:                 getKvGetterForStateTable(db, kv.EpochData),
-			kv.InactivityScores:          getKvGetterForStateTable(db, kv.InactivityScores),
-			kv.NextSyncCommittee:         getKvGetterForStateTable(db, kv.NextSyncCommittee),
-			kv.CurrentSyncCommittee:      getKvGetterForStateTable(db, kv.CurrentSyncCommittee),
-			kv.Eth1DataVotes:             getKvGetterForStateTable(db, kv.Eth1DataVotes),
-			kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
-			kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
-			kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
-			kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
-			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
+			// kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
+			// kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
+			// kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
+			// kv.StateEvents:               getKvGetterForStateTable(db, kv.StateEvents),
+			// kv.ActiveValidatorIndicies:   getKvGetterForStateTable(db, kv.ActiveValidatorIndicies),
+			// kv.StateRoot:                 getKvGetterForStateTable(db, kv.StateRoot),
+			kv.BlockRoot: getKvGetterForStateTable(db, kv.BlockRoot),
+			// kv.SlotData:                  getKvGetterForStateTable(db, kv.SlotData),
+			// kv.EpochData:                 getKvGetterForStateTable(db, kv.EpochData),
+			// kv.InactivityScores:          getKvGetterForStateTable(db, kv.InactivityScores),
+			// kv.NextSyncCommittee:         getKvGetterForStateTable(db, kv.NextSyncCommittee),
+			// kv.CurrentSyncCommittee:      getKvGetterForStateTable(db, kv.CurrentSyncCommittee),
+			// kv.Eth1DataVotes:             getKvGetterForStateTable(db, kv.Eth1DataVotes),
+			// kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
+			// kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
+			// kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
+			// kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
+			// kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
 		Compression: map[string]bool{},
 	}
@@ -529,6 +529,9 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		if err != nil {
 			return err
 		}
+		if i == 999424 && snapName == "blockroot" {
+			fmt.Printf("BlockRoot: %d %x\n", i, dump)
+		}
 		if i%20_000 == 0 {
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
 		}

From f1d87108522a628b164ccde0e99b9ac6d59534d5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 00:58:56 +0200
Subject: [PATCH 071/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index b8c32807d52..9f56de8b481 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -529,7 +529,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		if err != nil {
 			return err
 		}
-		if i == 999424 && snapName == "blockroot" {
+		if i == 999424 {
 			fmt.Printf("BlockRoot: %d %x\n", i, dump)
 		}
 		if i%20_000 == 0 {

From 8063667310c35b67f5a4ee0b15d6fe4a683c9d5d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:02:46 +0200
Subject: [PATCH 072/131] save

---
 cmd/capcli/cli.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 4574b4f1376..fe6c8cf53b0 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -1222,9 +1222,17 @@ func (c *DumpStateSnapshots) Run(ctx *Context) error {
 	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
+	r, _ := stateSn.Get(kv.BlockRoot, 999424)
+	fmt.Printf("%x\n", r)
 
 	if err := stateSn.DumpCaplinState(ctx, stateSn.BlocksAvailable(), to, c.StepSize, salt, dirs, runtime.NumCPU(), log.LvlInfo, log.Root()); err != nil {
 		return err
 	}
+	if err := stateSn.OpenFolder(); err != nil {
+		return err
+	}
+	r, _ = stateSn.Get(kv.BlockRoot, 999424)
+	fmt.Printf("%x\n", r)
+
 	return nil
 }

From a1b7331cbbbbb354a5d96657f1b8c9fb88941814 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:04:19 +0200
Subject: [PATCH 073/131] save

---
 .../freezeblocks/caplin_state_snapshots.go    | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 9f56de8b481..bb2bfc2aa39 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -66,24 +66,24 @@ func getKvGetterForStateTable(db kv.RoDB, tableName string) KeyValueGetter {
 func MakeCaplinStateSnapshotsTypes(db kv.RoDB) SnapshotTypes {
 	return SnapshotTypes{
 		KeyValueGetters: map[string]KeyValueGetter{
-			// kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
-			// kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
-			// kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
-			// kv.StateEvents:               getKvGetterForStateTable(db, kv.StateEvents),
-			// kv.ActiveValidatorIndicies:   getKvGetterForStateTable(db, kv.ActiveValidatorIndicies),
-			// kv.StateRoot:                 getKvGetterForStateTable(db, kv.StateRoot),
-			kv.BlockRoot: getKvGetterForStateTable(db, kv.BlockRoot),
-			// kv.SlotData:                  getKvGetterForStateTable(db, kv.SlotData),
-			// kv.EpochData:                 getKvGetterForStateTable(db, kv.EpochData),
-			// kv.InactivityScores:          getKvGetterForStateTable(db, kv.InactivityScores),
-			// kv.NextSyncCommittee:         getKvGetterForStateTable(db, kv.NextSyncCommittee),
-			// kv.CurrentSyncCommittee:      getKvGetterForStateTable(db, kv.CurrentSyncCommittee),
-			// kv.Eth1DataVotes:             getKvGetterForStateTable(db, kv.Eth1DataVotes),
-			// kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
-			// kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
-			// kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
-			// kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
-			// kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
+			kv.ValidatorEffectiveBalance: getKvGetterForStateTable(db, kv.ValidatorEffectiveBalance),
+			kv.ValidatorSlashings:        getKvGetterForStateTable(db, kv.ValidatorSlashings),
+			kv.ValidatorBalance:          getKvGetterForStateTable(db, kv.ValidatorBalance),
+			kv.StateEvents:               getKvGetterForStateTable(db, kv.StateEvents),
+			kv.ActiveValidatorIndicies:   getKvGetterForStateTable(db, kv.ActiveValidatorIndicies),
+			kv.StateRoot:                 getKvGetterForStateTable(db, kv.StateRoot),
+			kv.BlockRoot:                 getKvGetterForStateTable(db, kv.BlockRoot),
+			kv.SlotData:                  getKvGetterForStateTable(db, kv.SlotData),
+			kv.EpochData:                 getKvGetterForStateTable(db, kv.EpochData),
+			kv.InactivityScores:          getKvGetterForStateTable(db, kv.InactivityScores),
+			kv.NextSyncCommittee:         getKvGetterForStateTable(db, kv.NextSyncCommittee),
+			kv.CurrentSyncCommittee:      getKvGetterForStateTable(db, kv.CurrentSyncCommittee),
+			kv.Eth1DataVotes:             getKvGetterForStateTable(db, kv.Eth1DataVotes),
+			kv.IntraRandaoMixes:          getKvGetterForStateTable(db, kv.IntraRandaoMixes),
+			kv.RandaoMixes:               getKvGetterForStateTable(db, kv.RandaoMixes),
+			kv.Proposers:                 getKvGetterForStateTable(db, kv.Proposers),
+			kv.BalancesDump:              getKvGetterForStateTable(db, kv.BalancesDump),
+			kv.EffectiveBalancesDump:     getKvGetterForStateTable(db, kv.EffectiveBalancesDump),
 		},
 		Compression: map[string]bool{},
 	}

From ce25fd7e1792c9e50cc38d3189e0100149a0d009 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:07:02 +0200
Subject: [PATCH 074/131] save

---
 cl/persistence/state/state_accessors.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 881e147ce91..3aebf42816f 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -35,7 +35,7 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
 		if snapshot != nil {
-			v, err := snapshot.Get(table, uint64(binary.LittleEndian.Uint32(key)))
+			v, err := snapshot.Get(table, uint64(binary.BigEndian.Uint32(key)))
 			if err != nil {
 				return nil, err
 			}

From a2705094fc600ec04dcc1e42f909f5b2df1b6d0d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:11:28 +0200
Subject: [PATCH 075/131] save

---
 .../historical_states_reader/historical_states_reader.go     | 5 -----
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go    | 3 ---
 2 files changed, 8 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 193c8c502b3..d87daa8e637 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -308,11 +308,6 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 		currKeySlot = i
 		out.Set(int(currKeySlot%size), common.BytesToHash(v))
 		inserted++
-		if table == kv.BlockRoot {
-			if int(currKeySlot%size) == 0 {
-				fmt.Println(i, common.BytesToHash(v))
-			}
-		}
 		if inserted == needFromDB {
 			break
 		}
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index bb2bfc2aa39..579f5ed1f47 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -529,9 +529,6 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 		if err != nil {
 			return err
 		}
-		if i == 999424 {
-			fmt.Printf("BlockRoot: %d %x\n", i, dump)
-		}
 		if i%20_000 == 0 {
 			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
 		}

From 8be282fe2709a33908c7519f229095469e27357b Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:12:54 +0200
Subject: [PATCH 076/131] save

---
 cmd/capcli/cli.go | 124 +++++++++++++++++++++++-----------------------
 1 file changed, 63 insertions(+), 61 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index fe6c8cf53b0..a1fe673aa49 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -536,6 +536,7 @@ func (c *LoopSnapshots) Run(ctx *Context) error {
 type RetrieveHistoricalState struct {
 	chainCfg
 	outputFolder
+	withPPROF
 	CompareFile string `help:"compare file" default:""`
 	CompareSlot uint64 `help:"compare slot" default:"0"`
 	Out         string `help:"output file" default:""`
@@ -587,70 +588,71 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
-	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
-	start := time.Now()
-	haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)
-	if err != nil {
-		return err
-	}
-	endTime := time.Since(start)
-	hRoot, err := haveState.HashSSZ()
-	if err != nil {
-		return err
-	}
-	log.Info("Got state", "slot", haveState.Slot(), "root", libcommon.Hash(hRoot), "elapsed", endTime)
-
-	if err := haveState.InitBeaconState(); err != nil {
-		return err
-	}
-
-	v := haveState.Version()
-	// encode and decode the state
-	enc, err := haveState.EncodeSSZ(nil)
-	if err != nil {
-		return err
-	}
-	haveState = state.New(beaconConfig)
-	if err := haveState.DecodeSSZ(enc, int(v)); err != nil {
-		return err
-	}
-	if r.Out != "" {
-		// create file
-		if err := os.WriteFile(r.Out, enc, 0644); err != nil {
+	for {
+		hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
+		start := time.Now()
+		haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)
+		if err != nil {
 			return err
 		}
-	}
-
-	hRoot, err = haveState.HashSSZ()
-	if err != nil {
-		return err
-	}
-	if r.CompareFile == "" {
-		return nil
-	}
-	// Read the content of CompareFile in a []byte  without afero
-	rawBytes, err := os.ReadFile(r.CompareFile)
-	if err != nil {
-		return err
-	}
-	// Decode the []byte into a state
-	wantState := state.New(beaconConfig)
-	if err := wantState.DecodeSSZ(rawBytes, int(haveState.Version())); err != nil {
-		return err
-	}
-	wRoot, err := wantState.HashSSZ()
-	if err != nil {
-		return err
-	}
-	if hRoot != wRoot {
-		for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
-			if haveState.BlockRoots().Get(i) != wantState.BlockRoots().Get(i) {
-				log.Info("block roots mismatch", "index", i, "have", haveState.BlockRoots().Get(i), "want", wantState.BlockRoots().Get(i))
-			}
+		endTime := time.Since(start)
+		hRoot, err := haveState.HashSSZ()
+		if err != nil {
+			return err
 		}
-		return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot))
-	}
-	return nil
+		log.Info("Got state", "slot", haveState.Slot(), "root", libcommon.Hash(hRoot), "elapsed", endTime)
+	}
+	// if err := haveState.InitBeaconState(); err != nil {
+	// 	return err
+	// }
+
+	// v := haveState.Version()
+	// // encode and decode the state
+	// enc, err := haveState.EncodeSSZ(nil)
+	// if err != nil {
+	// 	return err
+	// }
+	// haveState = state.New(beaconConfig)
+	// if err := haveState.DecodeSSZ(enc, int(v)); err != nil {
+	// 	return err
+	// }
+	// if r.Out != "" {
+	// 	// create file
+	// 	if err := os.WriteFile(r.Out, enc, 0644); err != nil {
+	// 		return err
+	// 	}
+	// }
+
+	// hRoot, err = haveState.HashSSZ()
+	// if err != nil {
+	// 	return err
+	// }
+	// if r.CompareFile == "" {
+	// 	return nil
+	// }
+	// // Read the content of CompareFile in a []byte  without afero
+	// rawBytes, err := os.ReadFile(r.CompareFile)
+	// if err != nil {
+	// 	return err
+	// }
+	// // Decode the []byte into a state
+	// wantState := state.New(beaconConfig)
+	// if err := wantState.DecodeSSZ(rawBytes, int(haveState.Version())); err != nil {
+	// 	return err
+	// }
+	// wRoot, err := wantState.HashSSZ()
+	// if err != nil {
+	// 	return err
+	// }
+	// if hRoot != wRoot {
+	// 	for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
+	// 		if haveState.BlockRoots().Get(i) != wantState.BlockRoots().Get(i) {
+	// 			log.Info("block roots mismatch", "index", i, "have", haveState.BlockRoots().Get(i), "want", wantState.BlockRoots().Get(i))
+	// 		}
+	// 	}
+	// 	return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot))
+	// }
+	// return nil
 }
 
 type ArchiveSanitizer struct {

From 5a376f986ce892a578406dc71a7dde7960177470 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:15:03 +0200
Subject: [PATCH 077/131] save

---
 cmd/capcli/cli.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a1fe673aa49..099832c210c 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -588,6 +588,7 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
+	r.withPPROF.withProfile()
 	for {
 		hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
 		start := time.Now()

From 661d7bfaa70410db2744c59d0eefc4e2dbf73442 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:16:08 +0200
Subject: [PATCH 078/131] save

---
 cl/phase1/core/state/raw/hashing.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index 79a286cf01f..a4390593baf 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,7 +17,6 @@
 package raw
 
 import (
-	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -32,9 +31,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	for i := 0; i < len(b.leaves); i += 32 {
-		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	}
+	// for i := 0; i < len(b.leaves); i += 32 {
+	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	// }
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return

From c6810c5d61c419a54089e2ae8e98efbb8fc93815 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:35:16 +0200
Subject: [PATCH 079/131] save

---
 cl/persistence/state/state_accessors.go       | 15 ++++++++-----
 .../freezeblocks/block_snapshots.go           | 22 +++++++++++++++++++
 .../freezeblocks/caplin_state_snapshots.go    | 22 +------------------
 3 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 3aebf42816f..b232511b452 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -35,12 +35,15 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
 		if snapshot != nil {
-			v, err := snapshot.Get(table, uint64(binary.BigEndian.Uint32(key)))
-			if err != nil {
-				return nil, err
-			}
-			if v != nil {
-				return v, nil
+			view := snapshot.View()
+			// v, err := snapshot.Get(table, uint64(binary.BigEndian.Uint32(key)))
+			// if err != nil {
+			// 	return nil, err
+			// }
+			slot := uint64(binary.BigEndian.Uint32(key))
+			segment, ok := view.VisibleSegment(slot, table)
+			if ok {
+				return segment.Get(slot)
 			}
 		}
 		return tx.GetOne(table, key)
diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 7c8736e99b9..18169a85573 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -234,6 +234,28 @@ type VisibleSegment struct {
 	src     *DirtySegment
 }
 
+func (v *VisibleSegment) Get(globalId uint64) ([]byte, error) {
+	idxSlot := v.src.Index()
+
+	if idxSlot == nil {
+		return nil, nil
+	}
+	blockOffset := idxSlot.OrdinalLookup(globalId - idxSlot.BaseDataID())
+
+	gg := v.src.MakeGetter()
+	gg.Reset(blockOffset)
+	if !gg.HasNext() {
+		return nil, nil
+	}
+	var buf []byte
+	buf, _ = gg.Next(buf)
+	if len(buf) == 0 {
+		return nil, nil
+	}
+
+	return buf, nil
+}
+
 func DirtySegmentLess(i, j *DirtySegment) bool {
 	if i.from != j.from {
 		return i.from < j.from
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 579f5ed1f47..ebe6d2021b4 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -649,30 +649,10 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 	view := s.View()
 	defer view.Close()
 
-	var buf []byte
-
 	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
 	if !ok {
 		return nil, nil
 	}
 
-	idxSlot := seg.src.Index()
-
-	if idxSlot == nil {
-		return nil, nil
-	}
-	blockOffset := idxSlot.OrdinalLookup(slot - idxSlot.BaseDataID())
-
-	gg := seg.src.MakeGetter()
-	gg.Reset(blockOffset)
-	if !gg.HasNext() {
-		return nil, nil
-	}
-
-	buf, _ = gg.Next(buf)
-	if len(buf) == 0 {
-		return nil, nil
-	}
-
-	return buf, nil
+	return seg.Get(slot)
 }

From feac154e0cf24701fdec973b5dcd8775dfb2f985 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:54:45 +0200
Subject: [PATCH 080/131] save

---
 cl/beacon/handler/attestation_rewards.go      |  10 +-
 cl/beacon/handler/committees.go               |   7 +-
 cl/beacon/handler/duties_attester.go          |   8 +-
 cl/beacon/handler/duties_sync.go              |   4 +-
 cl/beacon/handler/lighthouse.go               |  18 +-
 cl/beacon/handler/liveness.go                 |   7 +-
 cl/beacon/handler/rewards.go                  |  10 +-
 cl/beacon/handler/states.go                   |  17 +-
 cl/beacon/handler/validators.go               |  25 ++-
 .../attesting_indicies.go                     |   3 +-
 .../historical_states_reader.go               | 154 +++++++-----------
 cl/persistence/state/state_accessors.go       |   7 +-
 12 files changed, 139 insertions(+), 131 deletions(-)

diff --git a/cl/beacon/handler/attestation_rewards.go b/cl/beacon/handler/attestation_rewards.go
index 4c09934754a..fa88d1ad9c5 100644
--- a/cl/beacon/handler/attestation_rewards.go
+++ b/cl/beacon/handler/attestation_rewards.go
@@ -178,15 +178,17 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 	if lastSlot > stateProgress {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("requested range is not yet processed or the node is not archivial"))
 	}
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
 
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 
 	epochData, err := state_accessors.ReadEpochData(stateGetter, a.beaconChainCfg.RoundSlotToEpoch(lastSlot))
 	if err != nil {
 		return nil, err
 	}
 
-	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, lastSlot)
+	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, stateGetter, lastSlot)
 	if err != nil {
 		return nil, err
 	}
@@ -194,7 +196,7 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("no validator set found for this epoch"))
 	}
 
-	_, previousIdx, err := a.stateReader.ReadParticipations(tx, lastSlot)
+	_, previousIdx, err := a.stateReader.ReadParticipations(tx, stateGetter, lastSlot)
 	if err != nil {
 		return nil, err
 	}
@@ -214,7 +216,7 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 		return resp.WithFinalized(true).WithOptimistic(a.forkchoiceStore.IsRootOptimistic(root)), nil
 	}
 	inactivityScores := solid.NewUint64ListSSZ(int(a.beaconChainCfg.ValidatorRegistryLimit))
-	if err := a.stateReader.ReconstructUint64ListDump(tx, lastSlot, kv.InactivityScores, validatorSet.Length(), inactivityScores); err != nil {
+	if err := a.stateReader.ReconstructUint64ListDump(stateGetter, lastSlot, kv.InactivityScores, validatorSet.Length(), inactivityScores); err != nil {
 		return nil, err
 	}
 	resp, err := a.computeAttestationsRewardsForAltair(
diff --git a/cl/beacon/handler/committees.go b/cl/beacon/handler/committees.go
index 6074879d492..35516f4c314 100644
--- a/cl/beacon/handler/committees.go
+++ b/cl/beacon/handler/committees.go
@@ -119,9 +119,12 @@ func (a *ApiHandler) getCommittees(w http.ResponseWriter, r *http.Request) (*bea
 		}
 		return newBeaconResponse(resp).WithFinalized(isFinalized).WithOptimistic(isOptimistic), nil
 	}
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	// finality case
 	activeIdxs, err := state_accessors.ReadActiveIndicies(
-		state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+		stateGetter,
 		epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
@@ -136,7 +139,7 @@ func (a *ApiHandler) getCommittees(w http.ResponseWriter, r *http.Request) (*bea
 	}
 
 	mixPosition := (epoch + a.beaconChainCfg.EpochsPerHistoricalVector - a.beaconChainCfg.MinSeedLookahead - 1) % a.beaconChainCfg.EpochsPerHistoricalVector
-	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, epoch*a.beaconChainCfg.SlotsPerEpoch, mixPosition)
+	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch, mixPosition)
 	if err != nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not read randao mix: %v", err))
 	}
diff --git a/cl/beacon/handler/duties_attester.go b/cl/beacon/handler/duties_attester.go
index 8452fbf3332..8c0ed8452be 100644
--- a/cl/beacon/handler/duties_attester.go
+++ b/cl/beacon/handler/duties_attester.go
@@ -150,9 +150,13 @@ func (a *ApiHandler) getAttesterDuties(w http.ResponseWriter, r *http.Request) (
 		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, fmt.Errorf("epoch %d is too far in the future", epoch))
 	}
 
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	// finality case
 	activeIdxs, err := state_accessors.ReadActiveIndicies(
-		state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+		stateGetter,
 		epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
@@ -167,7 +171,7 @@ func (a *ApiHandler) getAttesterDuties(w http.ResponseWriter, r *http.Request) (
 	}
 
 	mixPosition := (epoch + a.beaconChainCfg.EpochsPerHistoricalVector - a.beaconChainCfg.MinSeedLookahead - 1) % a.beaconChainCfg.EpochsPerHistoricalVector
-	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, epoch*a.beaconChainCfg.SlotsPerEpoch, mixPosition)
+	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch, mixPosition)
 	if err != nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not read randao mix: %v", err))
 	}
diff --git a/cl/beacon/handler/duties_sync.go b/cl/beacon/handler/duties_sync.go
index 2609d0b3103..bc4e7cc082a 100644
--- a/cl/beacon/handler/duties_sync.go
+++ b/cl/beacon/handler/duties_sync.go
@@ -81,10 +81,12 @@ func (a *ApiHandler) getSyncDuties(w http.ResponseWriter, r *http.Request) (*bea
 	if !ok {
 		_, syncCommittee, ok = a.forkchoiceStore.GetSyncCommittees(period - 1)
 	}
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
 	// Read them from the archive node if we do not have them in the fast-access storage
 	if !ok {
 		syncCommittee, err = state_accessors.ReadCurrentSyncCommittee(
-			state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots),
+			state_accessors.GetValFnTxAndSnapshot(tx, snRoTx),
 			a.beaconChainCfg.RoundSlotToSyncCommitteePeriod(startSlotAtEpoch))
 		if syncCommittee == nil {
 			log.Warn("could not find sync committee for epoch", "epoch", epoch, "period", period)
diff --git a/cl/beacon/handler/lighthouse.go b/cl/beacon/handler/lighthouse.go
index 10baeca3888..f2e978f5e56 100644
--- a/cl/beacon/handler/lighthouse.go
+++ b/cl/beacon/handler/lighthouse.go
@@ -75,7 +75,10 @@ func (a *ApiHandler) GetLighthouseValidatorInclusionGlobal(w http.ResponseWriter
 		return nil, err
 	}
 	defer tx.Rollback()
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 
 	slot := epoch * a.beaconChainCfg.SlotsPerEpoch
 	if slot >= a.forkchoiceStore.LowestAvailableSlot() {
@@ -135,15 +138,16 @@ func (a *ApiHandler) GetLighthouseValidatorInclusionGlobal(w http.ResponseWriter
 	if prevEpochData == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("epoch data not found for previous epoch"))
 	}
+
 	// read the validator set
-	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, slot)
+	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, stateGetter, slot)
 	if err != nil {
 		return nil, err
 	}
 	if validatorSet == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("validator set not found for current epoch"))
 	}
-	currentEpochParticipation, previousEpochParticipation, err := a.stateReader.ReadParticipations(tx, slot+(a.beaconChainCfg.SlotsPerEpoch-1))
+	currentEpochParticipation, previousEpochParticipation, err := a.stateReader.ReadParticipations(tx, stateGetter, slot+(a.beaconChainCfg.SlotsPerEpoch-1))
 	if err != nil {
 		return nil, err
 	}
@@ -278,7 +282,9 @@ func (a *ApiHandler) GetLighthouseValidatorInclusion(w http.ResponseWriter, r *h
 		return newBeaconResponse(a.computeLighthouseValidatorInclusion(int(validatorIndex), prevEpoch, epoch, activeBalance, prevActiveBalance, validatorSet, currentEpochParticipation, previousEpochParticipation)), nil
 	}
 
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	// read the epoch datas first
 	epochData, err := state_accessors.ReadEpochData(stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
@@ -295,14 +301,14 @@ func (a *ApiHandler) GetLighthouseValidatorInclusion(w http.ResponseWriter, r *h
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("epoch data not found for previous epoch"))
 	}
 	// read the validator set
-	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, slot)
+	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, stateGetter, slot)
 	if err != nil {
 		return nil, err
 	}
 	if validatorSet == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("validator set not found for current epoch"))
 	}
-	currentEpochParticipation, previousEpochParticipation, err := a.stateReader.ReadParticipations(tx, slot+(a.beaconChainCfg.SlotsPerEpoch-1))
+	currentEpochParticipation, previousEpochParticipation, err := a.stateReader.ReadParticipations(tx, stateGetter, slot+(a.beaconChainCfg.SlotsPerEpoch-1))
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/liveness.go b/cl/beacon/handler/liveness.go
index ccce105a571..f81d2e74621 100644
--- a/cl/beacon/handler/liveness.go
+++ b/cl/beacon/handler/liveness.go
@@ -28,6 +28,7 @@ import (
 	"github.com/erigontech/erigon/cl/beacon/beaconhttp"
 	"github.com/erigontech/erigon/cl/cltypes"
 	"github.com/erigontech/erigon/cl/cltypes/solid"
+	state_accessors "github.com/erigontech/erigon/cl/persistence/state"
 )
 
 type live struct {
@@ -138,11 +139,15 @@ func (a *ApiHandler) obtainCurrentEpochParticipationFromEpoch(tx kv.Tx, epoch ui
 	if epoch > 0 {
 		prevEpoch--
 	}
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 
 	currParticipation, ok1 := a.forkchoiceStore.Participation(epoch)
 	prevParticipation, ok2 := a.forkchoiceStore.Participation(prevEpoch)
 	if !ok1 || !ok2 {
-		return a.stateReader.ReadParticipations(tx, blockSlot)
+		return a.stateReader.ReadParticipations(tx, stateGetter, blockSlot)
 	}
 	return currParticipation, prevParticipation, nil
 
diff --git a/cl/beacon/handler/rewards.go b/cl/beacon/handler/rewards.go
index aacf6d90307..6a302207020 100644
--- a/cl/beacon/handler/rewards.go
+++ b/cl/beacon/handler/rewards.go
@@ -81,7 +81,10 @@ func (a *ApiHandler) GetEthV1BeaconRewardsBlocks(w http.ResponseWriter, r *http.
 			Total:             blkRewards.Attestations + blkRewards.ProposerSlashings + blkRewards.AttesterSlashings + blkRewards.SyncAggregate,
 		}).WithFinalized(isFinalized).WithOptimistic(isOptimistic), nil
 	}
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	slotData, err := state_accessors.ReadSlotData(stateGetter, slot)
 	if err != nil {
 		return nil, err
@@ -166,7 +169,10 @@ func (a *ApiHandler) PostEthV1BeaconRewardsSyncCommittees(w http.ResponseWriter,
 		syncCommittee      *solid.SyncCommittee
 		totalActiveBalance uint64
 	)
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	if slot < a.forkchoiceStore.LowestAvailableSlot() {
 		if !isCanonical {
 			return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("non-canonical finalized block not found"))
diff --git a/cl/beacon/handler/states.go b/cl/beacon/handler/states.go
index d5f8c66c6d7..751f1e8867f 100644
--- a/cl/beacon/handler/states.go
+++ b/cl/beacon/handler/states.go
@@ -263,7 +263,10 @@ func (a *ApiHandler) getFinalityCheckpoints(w http.ResponseWriter, r *http.Reque
 		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
 	}
 
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 	if !ok {
 		currentJustifiedCheckpoint, previousJustifiedCheckpoint, finalizedCheckpoint, ok, err = state_accessors.ReadCheckpoints(stateGetter, a.beaconChainCfg.RoundSlotToEpoch(*slot))
 		if err != nil {
@@ -318,7 +321,11 @@ func (a *ApiHandler) getSyncCommittees(w http.ResponseWriter, r *http.Request) (
 	if slot == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not read block slot: %x", blockRoot))
 	}
-	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, a.caplinStateSnapshots)
+
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
 
 	// Code here
 	currentSyncCommittee, nextSyncCommittee, ok := a.forkchoiceStore.GetSyncCommittees(a.beaconChainCfg.SyncCommitteePeriod(*slot))
@@ -444,7 +451,11 @@ func (a *ApiHandler) getRandao(w http.ResponseWriter, r *http.Request) (*beaconh
 	if canonicalRoot != blockRoot {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, fmt.Errorf("could not read randao: %x", blockRoot))
 	}
-	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, slot, epoch%a.beaconChainCfg.EpochsPerHistoricalVector)
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
+	mix, err := a.stateReader.ReadRandaoMixBySlotAndIndex(tx, stateGetter, slot, epoch%a.beaconChainCfg.EpochsPerHistoricalVector)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cl/beacon/handler/validators.go b/cl/beacon/handler/validators.go
index 9b76e6d24e3..39e20896d88 100644
--- a/cl/beacon/handler/validators.go
+++ b/cl/beacon/handler/validators.go
@@ -338,8 +338,13 @@ func (a *ApiHandler) writeValidatorsResponse(
 	}
 	stateEpoch := *slot / a.beaconChainCfg.SlotsPerEpoch
 
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
+
 	if *slot < a.forkchoiceStore.LowestAvailableSlot() {
-		validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, *slot)
+		validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, getter, *slot)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -347,7 +352,7 @@ func (a *ApiHandler) writeValidatorsResponse(
 			http.Error(w, fmt.Errorf("state not found for slot %v", *slot).Error(), http.StatusNotFound)
 			return
 		}
-		balances, err := a.stateReader.ReadValidatorsBalances(tx, *slot)
+		balances, err := a.stateReader.ReadValidatorsBalances(tx, getter, *slot)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -454,6 +459,11 @@ func (a *ApiHandler) GetEthV1BeaconStatesValidator(w http.ResponseWriter, r *htt
 		return nil, err
 	}
 
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
+
 	if blockId.Head() { // Lets see if we point to head, if yes then we need to look at the head state we always keep.
 		s := a.syncedData.HeadState()
 		if s == nil {
@@ -475,14 +485,14 @@ func (a *ApiHandler) GetEthV1BeaconStatesValidator(w http.ResponseWriter, r *htt
 	stateEpoch := *slot / a.beaconChainCfg.SlotsPerEpoch
 
 	if *slot < a.forkchoiceStore.LowestAvailableSlot() {
-		validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, *slot)
+		validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, getter, *slot)
 		if err != nil {
 			return nil, err
 		}
 		if validatorSet == nil {
 			return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("validators not found"))
 		}
-		balances, err := a.stateReader.ReadValidatorsBalances(tx, *slot)
+		balances, err := a.stateReader.ReadValidatorsBalances(tx, getter, *slot)
 		if err != nil {
 			return nil, err
 		}
@@ -595,8 +605,13 @@ func (a *ApiHandler) getValidatorBalances(ctx context.Context, w http.ResponseWr
 		return
 	}
 
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
+
+	getter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
+
 	if *slot < a.forkchoiceStore.LowestAvailableSlot() {
-		balances, err := a.stateReader.ReadValidatorsBalances(tx, *slot)
+		balances, err := a.stateReader.ReadValidatorsBalances(tx, getter, *slot)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
diff --git a/cl/persistence/state/historical_states_reader/attesting_indicies.go b/cl/persistence/state/historical_states_reader/attesting_indicies.go
index d02ab0d7023..14001ba3808 100644
--- a/cl/persistence/state/historical_states_reader/attesting_indicies.go
+++ b/cl/persistence/state/historical_states_reader/attesting_indicies.go
@@ -163,8 +163,7 @@ func (r *HistoricalStatesReader) readHistoricalBlockRoot(tx kv.Tx, slot, index u
 
 }
 
-func (r *HistoricalStatesReader) getAttestationParticipationFlagIndicies(tx kv.Tx, version clparams.StateVersion, stateSlot uint64, data solid.AttestationData, inclusionDelay uint64, skipAssert bool) ([]uint8, error) {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+func (r *HistoricalStatesReader) getAttestationParticipationFlagIndicies(tx kv.Tx, getter state_accessors.GetValFn, version clparams.StateVersion, stateSlot uint64, data solid.AttestationData, inclusionDelay uint64, skipAssert bool) ([]uint8, error) {
 
 	currentCheckpoint, previousCheckpoint, _, ok, err := state_accessors.ReadCheckpoints(getter, r.cfg.RoundSlotToEpoch(stateSlot))
 	if err != nil {
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index d87daa8e637..1649a1f98c7 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -75,7 +75,9 @@ func NewHistoricalStatesReader(
 }
 
 func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.Tx, slot uint64) (*state.CachingBeaconState, error) {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+	snapshotView := r.stateSn.View()
+	defer snapshotView.Close()
+	kvGetter := state_accessors.GetValFnTxAndSnapshot(tx, snapshotView)
 
 	ret := state.New(r.cfg)
 	latestProcessedState, err := state_accessors.GetStateProcessingProgress(tx)
@@ -105,7 +107,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	blockHeader := block.SignedBeaconBlockHeader().Header
 	blockHeader.Root = common.Hash{}
 	// Read the epoch and per-slot data.
-	slotData, err := state_accessors.ReadSlotData(getter, slot)
+	slotData, err := state_accessors.ReadSlotData(kvGetter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -115,7 +117,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	}
 	roundedSlot := r.cfg.RoundSlotToEpoch(slot)
 
-	epochData, err := state_accessors.ReadEpochData(getter, roundedSlot)
+	epochData, err := state_accessors.ReadEpochData(kvGetter, roundedSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read epoch data: %w", err)
 	}
@@ -134,12 +136,12 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	stateRoots, blockRoots := solid.NewHashVector(int(r.cfg.SlotsPerHistoricalRoot)), solid.NewHashVector(int(r.cfg.SlotsPerHistoricalRoot))
 	ret.SetLatestBlockHeader(blockHeader)
 
-	if err := r.readHistoryHashVector(tx, r.genesisState.BlockRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.BlockRoot, blockRoots); err != nil {
+	if err := r.readHistoryHashVector(tx, kvGetter, r.genesisState.BlockRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.BlockRoot, blockRoots); err != nil {
 		return nil, fmt.Errorf("failed to read block roots: %w", err)
 	}
 	ret.SetBlockRoots(blockRoots)
 
-	if err := r.readHistoryHashVector(tx, r.genesisState.StateRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.StateRoot, stateRoots); err != nil {
+	if err := r.readHistoryHashVector(tx, kvGetter, r.genesisState.StateRoots(), slot, r.cfg.SlotsPerHistoricalRoot, kv.StateRoot, stateRoots); err != nil {
 		return nil, fmt.Errorf("failed to read state roots: %w", err)
 	}
 	ret.SetStateRoots(stateRoots)
@@ -155,14 +157,14 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 
 	// Eth1
 	eth1DataVotes := solid.NewStaticListSSZ[*cltypes.Eth1Data](int(r.cfg.Eth1DataVotesLength()), 72)
-	if err := r.readEth1DataVotes(tx, slotData.Eth1DataLength, slot, eth1DataVotes); err != nil {
+	if err := r.readEth1DataVotes(kvGetter, slotData.Eth1DataLength, slot, eth1DataVotes); err != nil {
 		return nil, fmt.Errorf("failed to read eth1 data votes: %w", err)
 	}
 	ret.SetEth1DataVotes(eth1DataVotes)
 	ret.SetEth1Data(slotData.Eth1Data)
 	ret.SetEth1DepositIndex(slotData.Eth1DepositIndex)
 	// Registry (Validators + Balances)
-	balancesBytes, err := r.reconstructBalances(tx, slotData.ValidatorLength, slot, kv.ValidatorBalance, kv.BalancesDump)
+	balancesBytes, err := r.reconstructBalances(tx, kvGetter, slotData.ValidatorLength, slot, kv.ValidatorBalance, kv.BalancesDump)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read validator balances: %w", err)
 	}
@@ -173,27 +175,27 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 
 	ret.SetBalances(balances)
 
-	validatorSet, err := r.ReadValidatorsForHistoricalState(tx, slot)
+	validatorSet, err := r.ReadValidatorsForHistoricalState(tx, kvGetter, slot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read validators: %w", err)
 	}
 	ret.SetValidators(validatorSet)
 	// Randomness
 	randaoMixes := solid.NewHashVector(int(r.cfg.EpochsPerHistoricalVector))
-	if err := r.readRandaoMixes(tx, slot, randaoMixes); err != nil {
+	if err := r.readRandaoMixes(tx, kvGetter, slot, randaoMixes); err != nil {
 		return nil, fmt.Errorf("failed to read randao mixes: %w", err)
 	}
 	ret.SetRandaoMixes(randaoMixes)
 	slashingsVector := solid.NewUint64VectorSSZ(int(r.cfg.EpochsPerSlashingsVector))
 	// Slashings
-	err = r.ReconstructUint64ListDump(tx, slot, kv.ValidatorSlashings, int(r.cfg.EpochsPerSlashingsVector), slashingsVector)
+	err = r.ReconstructUint64ListDump(kvGetter, slot, kv.ValidatorSlashings, int(r.cfg.EpochsPerSlashingsVector), slashingsVector)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read slashings: %w", err)
 	}
 	ret.SetSlashings(slashingsVector)
 
 	// Finality
-	currentCheckpoint, previousCheckpoint, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(getter, roundedSlot)
+	currentCheckpoint, previousCheckpoint, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(kvGetter, roundedSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read checkpoints: %w", err)
 	}
@@ -216,7 +218,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 		ret.SetCurrentEpochAttestations(currentAtts)
 		ret.SetPreviousEpochAttestations(previousAtts)
 	} else {
-		currentIdxs, previousIdxs, err := r.ReadParticipations(tx, slot)
+		currentIdxs, previousIdxs, err := r.ReadParticipations(tx, kvGetter, slot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read participations: %w", err)
 		}
@@ -229,7 +231,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	}
 	inactivityScores := solid.NewUint64ListSSZ(int(r.cfg.ValidatorRegistryLimit))
 	// Inactivity
-	err = r.ReconstructUint64ListDump(tx, slot, kv.InactivityScores, int(slotData.ValidatorLength), inactivityScores)
+	err = r.ReconstructUint64ListDump(kvGetter, slot, kv.InactivityScores, int(slotData.ValidatorLength), inactivityScores)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read inactivity scores: %w", err)
 	}
@@ -237,7 +239,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	ret.SetInactivityScoresRaw(inactivityScores)
 	// Sync
 	syncCommitteeSlot := r.cfg.RoundSlotToSyncCommitteePeriod(slot)
-	currentSyncCommittee, err := state_accessors.ReadCurrentSyncCommittee(getter, syncCommitteeSlot)
+	currentSyncCommittee, err := state_accessors.ReadCurrentSyncCommittee(kvGetter, syncCommitteeSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read current sync committee: %w", err)
 	}
@@ -245,7 +247,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 		currentSyncCommittee = r.genesisState.CurrentSyncCommittee()
 	}
 
-	nextSyncCommittee, err := state_accessors.ReadNextSyncCommittee(getter, syncCommitteeSlot)
+	nextSyncCommittee, err := state_accessors.ReadNextSyncCommittee(kvGetter, syncCommitteeSlot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read next sync committee: %w", err)
 	}
@@ -282,7 +284,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	return ret, nil
 }
 
-func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector solid.HashVectorSSZ, slot, size uint64, table string, out solid.HashVectorSSZ) (err error) {
+func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, kvGetter state_accessors.GetValFn, genesisVector solid.HashVectorSSZ, slot, size uint64, table string, out solid.HashVectorSSZ) (err error) {
 	var needFromGenesis, inserted uint64
 	if size > slot || slot-size <= r.genesisState.Slot() {
 		needFromGenesis = size - (slot - r.genesisState.Slot())
@@ -295,10 +297,9 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 	}
 
 	var currKeySlot uint64
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 	for i := slot - needFromDB; i <= highestAvaiableSlot; i++ {
 		key := base_encoding.Encode64ToBytes4(i)
-		v, err := getter(table, key)
+		v, err := kvGetter(table, key)
 		if err != nil {
 			return err
 		}
@@ -320,7 +321,7 @@ func (r *HistoricalStatesReader) readHistoryHashVector(tx kv.Tx, genesisVector s
 	return nil
 }
 
-func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength, slot uint64, out *solid.ListSSZ[*cltypes.Eth1Data]) error {
+func (r *HistoricalStatesReader) readEth1DataVotes(kvGetter state_accessors.GetValFn, eth1DataVotesLength, slot uint64, out *solid.ListSSZ[*cltypes.Eth1Data]) error {
 	initialSlot := r.cfg.RoundSlotToVotePeriod(slot)
 	if initialSlot <= r.genesisState.Slot() {
 		// We need to prepend the genesis votes
@@ -329,7 +330,6 @@ func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength
 		}
 	}
 
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 	endSlot := r.cfg.RoundSlotToVotePeriod(slot + r.cfg.SlotsPerEpoch*r.cfg.EpochsPerEth1VotingPeriod)
 
 	for i := initialSlot; i < endSlot; i++ {
@@ -337,7 +337,7 @@ func (r *HistoricalStatesReader) readEth1DataVotes(tx kv.Tx, eth1DataVotesLength
 			break
 		}
 		key := base_encoding.Encode64ToBytes4(i)
-		v, err := getter(kv.Eth1DataVotes, key)
+		v, err := kvGetter(kv.Eth1DataVotes, key)
 		if err != nil {
 			return err
 		}
@@ -378,7 +378,7 @@ func (r *HistoricalStatesReader) highestSlotInSnapshotsAndDB(tx kv.Tx, tbl strin
 	return max(avaiableInDB, availableInSnapshots), nil
 }
 
-func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out solid.HashVectorSSZ) error {
+func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, kvGetter state_accessors.GetValFn, slot uint64, out solid.HashVectorSSZ) error {
 	size := r.cfg.EpochsPerHistoricalVector
 	genesisVector := r.genesisState.RandaoMixes()
 	var needFromGenesis, inserted uint64
@@ -388,7 +388,6 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 	if size > epoch || epoch-size <= genesisEpoch {
 		needFromGenesis = size - (epoch - genesisEpoch)
 	}
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	needFromDB := size - needFromGenesis
 
@@ -400,7 +399,7 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 
 	for i := roundedSlot - (needFromDB)*r.cfg.SlotsPerEpoch; i <= highestAvaiableSlot; i++ {
 		key := base_encoding.Encode64ToBytes4(i)
-		v, err := getter(kv.RandaoMixes, key)
+		v, err := kvGetter(kv.RandaoMixes, key)
 		if err != nil {
 			return err
 		}
@@ -423,7 +422,7 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 	}
 
 	// Now we need to read the intra epoch randao mix.
-	intraRandaoMix, err := getter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
+	intraRandaoMix, err := kvGetter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return err
 	}
@@ -434,11 +433,10 @@ func (r *HistoricalStatesReader) readRandaoMixes(tx kv.Tx, slot uint64, out soli
 	return nil
 }
 
-func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validatorSetLength, slot uint64, diffBucket string, dumpBucket string) ([]byte, error) {
+func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, kvGetter state_accessors.GetValFn, validatorSetLength, slot uint64, diffBucket string, dumpBucket string) ([]byte, error) {
 	// Read the file
 	remainder := slot % clparams.SlotsPerDump
 	freshDumpSlot := slot - remainder
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	midpoint := uint64(clparams.SlotsPerDump / 2)
 	var compressed []byte
@@ -448,12 +446,12 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 	}
 	forward := remainder <= midpoint || currentStageProgress <= freshDumpSlot+clparams.SlotsPerDump
 	if forward {
-		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
+		compressed, err = kvGetter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
+		compressed, err = kvGetter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
 		if err != nil {
 			return nil, err
 		}
@@ -487,28 +485,9 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 		return nil, err
 	}
 	if forward {
-		// for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) <= slot; k, v, err = diffCursor.Next() {
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// 	if len(k) != 4 {
-		// 		return nil, fmt.Errorf("invalid key %x", k)
-		// 	}
-		// 	currSlot := base_encoding.Decode64FromBytes4(k)
-		// 	if currSlot == freshDumpSlot {
-		// 		continue
-		// 	}
-		// 	if currSlot > slot {
-		// 		return nil, fmt.Errorf("diff not found for slot %d", slot)
-		// 	}
-		// 	currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, false)
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// }
 		for currSlot := freshDumpSlot; currSlot <= slot && currSlot < highestSlotAvailable; currSlot++ {
 			key := base_encoding.Encode64ToBytes4(currSlot)
-			v, err := getter(diffBucket, key)
+			v, err := kvGetter(diffBucket, key)
 			if err != nil {
 				return nil, err
 			}
@@ -527,25 +506,9 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 			}
 		}
 	} else {
-		// for k, v, err := diffCursor.Seek(base_encoding.Encode64ToBytes4(freshDumpSlot + clparams.SlotsPerDump)); err == nil && k != nil && base_encoding.Decode64FromBytes4(k) > slot; k, v, err = diffCursor.Prev() {
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// 	if len(k) != 4 {
-		// 		return nil, fmt.Errorf("invalid key %x", k)
-		// 	}
-		// 	currSlot := base_encoding.Decode64FromBytes4(k)
-		// 	if currSlot <= slot || currSlot > freshDumpSlot+clparams.SlotsPerDump {
-		// 		continue
-		// 	}
-		// 	currentList, err = base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, v, true)
-		// 	if err != nil {
-		// 		return nil, err
-		// 	}
-		// }
 		for currSlot := freshDumpSlot + clparams.SlotsPerDump; currSlot > slot && currSlot > r.genesisState.Slot(); currSlot-- {
 			key := base_encoding.Encode64ToBytes4(currSlot)
-			v, err := getter(diffBucket, key)
+			v, err := kvGetter(diffBucket, key)
 			if err != nil {
 				return nil, err
 			}
@@ -565,10 +528,9 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, validator
 	return currentList, err
 }
 
-func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLength, slot uint64, diffBucket, dumpBucket string) ([]byte, error) {
+func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, kvGetter state_accessors.GetValFn, validatorSetLength, slot uint64, diffBucket, dumpBucket string) ([]byte, error) {
 	remainder := slot % clparams.SlotsPerDump
 	freshDumpSlot := slot - remainder
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
 	buffer := buffersPool.Get().(*bytes.Buffer)
 	defer buffersPool.Put(buffer)
@@ -582,12 +544,12 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 	midpoint := uint64(clparams.SlotsPerDump / 2)
 	forward := remainder <= midpoint || currentStageProgress <= freshDumpSlot+clparams.SlotsPerDump
 	if forward {
-		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
+		compressed, err = kvGetter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		compressed, err = getter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
+		compressed, err = kvGetter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot+clparams.SlotsPerDump))
 		if err != nil {
 			return nil, err
 		}
@@ -616,7 +578,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 			if i == freshDumpSlot {
 				continue
 			}
-			diff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(i))
+			diff, err := kvGetter(diffBucket, base_encoding.Encode64ToBytes4(i))
 			if err != nil {
 				return nil, err
 			}
@@ -630,7 +592,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 		}
 	} else {
 		for i := freshDumpSlot + clparams.SlotsPerDump; i > roundedSlot; i -= r.cfg.SlotsPerEpoch {
-			diff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(i))
+			diff, err := kvGetter(diffBucket, base_encoding.Encode64ToBytes4(i))
 			if err != nil {
 				return nil, err
 			}
@@ -649,7 +611,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 		return currentList, nil
 	}
 
-	slotDiff, err := getter(diffBucket, base_encoding.Encode64ToBytes4(slot))
+	slotDiff, err := kvGetter(diffBucket, base_encoding.Encode64ToBytes4(slot))
 	if err != nil {
 		return nil, err
 	}
@@ -661,8 +623,7 @@ func (r *HistoricalStatesReader) reconstructBalances(tx kv.Tx, validatorSetLengt
 	return base_encoding.ApplyCompressedSerializedUint64ListDiff(currentList, currentList, slotDiff, false)
 }
 
-func (r *HistoricalStatesReader) ReconstructUint64ListDump(tx kv.Tx, slot uint64, bkt string, size int, out solid.Uint64ListSSZ) error {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+func (r *HistoricalStatesReader) ReconstructUint64ListDump(kvGetter state_accessors.GetValFn, slot uint64, bkt string, size int, out solid.Uint64ListSSZ) error {
 	var (
 		v   []byte
 		err error
@@ -670,7 +631,7 @@ func (r *HistoricalStatesReader) ReconstructUint64ListDump(tx kv.Tx, slot uint64
 	// Try seeking <= to slot
 	for i := slot; i >= r.genesisState.Slot(); i-- {
 		key := base_encoding.Encode64ToBytes4(i)
-		v, err = getter(bkt, key)
+		v, err = kvGetter(bkt, key)
 		if err != nil {
 			return err
 		}
@@ -699,11 +660,9 @@ func (r *HistoricalStatesReader) ReconstructUint64ListDump(tx kv.Tx, slot uint64
 	return out.DecodeSSZ(currentList, 0)
 }
 
-func (r *HistoricalStatesReader) ReadValidatorsForHistoricalState(tx kv.Tx, slot uint64) (*solid.ValidatorSet, error) {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
-
+func (r *HistoricalStatesReader) ReadValidatorsForHistoricalState(tx kv.Tx, kvGetter state_accessors.GetValFn, slot uint64) (*solid.ValidatorSet, error) {
 	// Read the minimal beacon state which have the small fields.
-	sd, err := state_accessors.ReadSlotData(getter, slot)
+	sd, err := state_accessors.ReadSlotData(kvGetter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -724,7 +683,7 @@ func (r *HistoricalStatesReader) ReadValidatorsForHistoricalState(tx kv.Tx, slot
 	})
 	// Read the balances
 
-	bytesEffectiveBalances, err := r.reconstructDiffedUint64List(tx, validatorSetLength, slot, kv.ValidatorEffectiveBalance, kv.EffectiveBalancesDump)
+	bytesEffectiveBalances, err := r.reconstructDiffedUint64List(tx, kvGetter, validatorSetLength, slot, kv.ValidatorEffectiveBalance, kv.EffectiveBalancesDump)
 	if err != nil {
 		return nil, err
 	}
@@ -787,13 +746,12 @@ func (r *HistoricalStatesReader) readPendingEpochs(tx kv.Tx, slot uint64) (*soli
 }
 
 // readParticipations shuffles active indicies and returns the participation flags for the given epoch.
-func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*solid.ParticipationBitList, *solid.ParticipationBitList, error) {
+func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, kvGetter state_accessors.GetValFn, slot uint64) (*solid.ParticipationBitList, *solid.ParticipationBitList, error) {
 	var beginSlot uint64
 	epoch, prevEpoch := r.computeRelevantEpochs(slot)
 	beginSlot = prevEpoch * r.cfg.SlotsPerEpoch
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
 
-	currentActiveIndicies, err := state_accessors.ReadActiveIndicies(getter, epoch*r.cfg.SlotsPerEpoch)
+	currentActiveIndicies, err := state_accessors.ReadActiveIndicies(kvGetter, epoch*r.cfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -801,14 +759,14 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 	if epoch == 0 {
 		previousActiveIndicies = currentActiveIndicies
 	} else {
-		previousActiveIndicies, err = state_accessors.ReadActiveIndicies(getter, (epoch-1)*r.cfg.SlotsPerEpoch)
+		previousActiveIndicies, err = state_accessors.ReadActiveIndicies(kvGetter, (epoch-1)*r.cfg.SlotsPerEpoch)
 		if err != nil {
 			return nil, nil, err
 		}
 	}
 
 	// Read the minimal beacon state which have the small fields.
-	sd, err := state_accessors.ReadSlotData(getter, slot)
+	sd, err := state_accessors.ReadSlotData(kvGetter, slot)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -824,7 +782,7 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 		return nil, nil, err
 	}
 	// trigger the cache for shuffled sets in parallel
-	if err := r.tryCachingEpochsInParallell(tx, [][]uint64{currentActiveIndicies, previousActiveIndicies}, []uint64{epoch, prevEpoch}); err != nil {
+	if err := r.tryCachingEpochsInParallell(tx, kvGetter, [][]uint64{currentActiveIndicies, previousActiveIndicies}, []uint64{epoch, prevEpoch}); err != nil {
 		return nil, nil, err
 	}
 	// Read the previous idxs
@@ -861,7 +819,7 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 			attestationEpoch := data.Slot / r.cfg.SlotsPerEpoch
 
 			mixPosition := (attestationEpoch + r.cfg.EpochsPerHistoricalVector - r.cfg.MinSeedLookahead - 1) % r.cfg.EpochsPerHistoricalVector
-			mix, err := r.ReadRandaoMixBySlotAndIndex(tx, data.Slot, mixPosition)
+			mix, err := r.ReadRandaoMixBySlotAndIndex(tx, kvGetter, data.Slot, mixPosition)
 			if err != nil {
 				return false
 			}
@@ -872,7 +830,7 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, slot uint64) (*sol
 				return false
 			}
 			var participationFlagsIndicies []uint8
-			participationFlagsIndicies, err = r.getAttestationParticipationFlagIndicies(tx, block.Version(), i, *data, i-data.Slot, true)
+			participationFlagsIndicies, err = r.getAttestationParticipationFlagIndicies(tx, kvGetter, block.Version(), i, *data, i-data.Slot, true)
 			if err != nil {
 				return false
 			}
@@ -913,12 +871,12 @@ func (r *HistoricalStatesReader) computeRelevantEpochs(slot uint64) (uint64, uin
 	return epoch, epoch - 1
 }
 
-func (r *HistoricalStatesReader) tryCachingEpochsInParallell(tx kv.Tx, activeIdxs [][]uint64, epochs []uint64) error {
+func (r *HistoricalStatesReader) tryCachingEpochsInParallell(tx kv.Tx, kvGetter state_accessors.GetValFn, activeIdxs [][]uint64, epochs []uint64) error {
 	var wg sync.WaitGroup
 	wg.Add(len(epochs))
 	for i, epoch := range epochs {
 		mixPosition := (epoch + r.cfg.EpochsPerHistoricalVector - r.cfg.MinSeedLookahead - 1) % r.cfg.EpochsPerHistoricalVector
-		mix, err := r.ReadRandaoMixBySlotAndIndex(tx, epochs[0]*r.cfg.SlotsPerEpoch, mixPosition)
+		mix, err := r.ReadRandaoMixBySlotAndIndex(tx, kvGetter, epochs[0]*r.cfg.SlotsPerEpoch, mixPosition)
 		if err != nil {
 			return err
 		}
@@ -933,9 +891,8 @@ func (r *HistoricalStatesReader) tryCachingEpochsInParallell(tx kv.Tx, activeIdx
 	return nil
 }
 
-func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, slot uint64) (solid.Uint64ListSSZ, error) {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
-	sd, err := state_accessors.ReadSlotData(getter, slot)
+func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, kvGetter state_accessors.GetValFn, slot uint64) (solid.Uint64ListSSZ, error) {
+	sd, err := state_accessors.ReadSlotData(kvGetter, slot)
 	if err != nil {
 		return nil, err
 	}
@@ -944,7 +901,7 @@ func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, slot uint64) (
 		return nil, nil
 	}
 
-	balances, err := r.reconstructBalances(tx, sd.ValidatorLength, slot, kv.ValidatorBalance, kv.BalancesDump)
+	balances, err := r.reconstructBalances(tx, kvGetter, sd.ValidatorLength, slot, kv.ValidatorBalance, kv.BalancesDump)
 	if err != nil {
 		return nil, err
 	}
@@ -953,12 +910,11 @@ func (r *HistoricalStatesReader) ReadValidatorsBalances(tx kv.Tx, slot uint64) (
 	return balancesList, balancesList.DecodeSSZ(balances, 0)
 }
 
-func (r *HistoricalStatesReader) ReadRandaoMixBySlotAndIndex(tx kv.Tx, slot, index uint64) (common.Hash, error) {
-	getter := state_accessors.GetValFnTxAndSnapshot(tx, r.stateSn)
+func (r *HistoricalStatesReader) ReadRandaoMixBySlotAndIndex(tx kv.Tx, kvGetter state_accessors.GetValFn, slot, index uint64) (common.Hash, error) {
 	epoch := slot / r.cfg.SlotsPerEpoch
 	epochSubIndex := epoch % r.cfg.EpochsPerHistoricalVector
 	if index == epochSubIndex {
-		intraRandaoMix, err := getter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
+		intraRandaoMix, err := kvGetter(kv.IntraRandaoMixes, base_encoding.Encode64ToBytes4(slot))
 		if err != nil {
 			return common.Hash{}, err
 		}
@@ -987,7 +943,7 @@ func (r *HistoricalStatesReader) ReadRandaoMixBySlotAndIndex(tx kv.Tx, slot, ind
 	if needFromGenesis {
 		return r.genesisState.GetRandaoMixes(epoch), nil
 	}
-	mixBytes, err := getter(kv.RandaoMixes, base_encoding.Encode64ToBytes4(epochLookup*r.cfg.SlotsPerEpoch))
+	mixBytes, err := kvGetter(kv.RandaoMixes, base_encoding.Encode64ToBytes4(epochLookup*r.cfg.SlotsPerEpoch))
 	if err != nil {
 		return common.Hash{}, err
 	}
diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index b232511b452..44a8bc9815f 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -32,16 +32,15 @@ import (
 
 type GetValFn func(table string, key []byte) ([]byte, error)
 
-func GetValFnTxAndSnapshot(tx kv.Tx, snapshot *freezeblocks.CaplinStateSnapshots) GetValFn {
+func GetValFnTxAndSnapshot(tx kv.Tx, snapshotRoTx *freezeblocks.CaplinStateView) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
-		if snapshot != nil {
-			view := snapshot.View()
+		if snapshotRoTx != nil {
 			// v, err := snapshot.Get(table, uint64(binary.BigEndian.Uint32(key)))
 			// if err != nil {
 			// 	return nil, err
 			// }
 			slot := uint64(binary.BigEndian.Uint32(key))
-			segment, ok := view.VisibleSegment(slot, table)
+			segment, ok := snapshotRoTx.VisibleSegment(slot, table)
 			if ok {
 				return segment.Get(slot)
 			}

From a96de6b7d903d9237c7c936a1ea3f25612a5f77d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 01:58:56 +0200
Subject: [PATCH 081/131] save

---
 .../historical_states_reader/historical_states_reader.go      | 1 +
 cl/persistence/state/state_accessors.go                       | 4 ----
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go     | 4 ++++
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 1649a1f98c7..2ccf35e5c98 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -77,6 +77,7 @@ func NewHistoricalStatesReader(
 func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.Tx, slot uint64) (*state.CachingBeaconState, error) {
 	snapshotView := r.stateSn.View()
 	defer snapshotView.Close()
+
 	kvGetter := state_accessors.GetValFnTxAndSnapshot(tx, snapshotView)
 
 	ret := state.New(r.cfg)
diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 44a8bc9815f..6d4ca4d6c10 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -35,10 +35,6 @@ type GetValFn func(table string, key []byte) ([]byte, error)
 func GetValFnTxAndSnapshot(tx kv.Tx, snapshotRoTx *freezeblocks.CaplinStateView) GetValFn {
 	return func(table string, key []byte) ([]byte, error) {
 		if snapshotRoTx != nil {
-			// v, err := snapshot.Get(table, uint64(binary.BigEndian.Uint32(key)))
-			// if err != nil {
-			// 	return nil, err
-			// }
 			slot := uint64(binary.BigEndian.Uint32(key))
 			segment, ok := snapshotRoTx.VisibleSegment(slot, table)
 			if ok {
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index ebe6d2021b4..f86732e20bf 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -493,10 +493,14 @@ func (v *CaplinStateView) Close() {
 }
 
 func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
+	if v.s == nil || v.s.Segments[tbl] == nil {
+		return nil
+	}
 	return v.s.Segments[tbl].VisibleSegments
 }
 
 func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {
+	tbl = strings.ToLower(tbl)
 	for _, seg := range v.VisibleSegments(tbl) {
 		if !(slot >= seg.from && slot < seg.to) {
 			continue

From 35913bcfa787c3630103078d9dd0f0829baf1087 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:02:24 +0200
Subject: [PATCH 082/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index f86732e20bf..c6901363dc4 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -139,7 +139,7 @@ func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.B
 	// }
 	Segments := make(map[string]*segments)
 	for k := range snapshotTypes.KeyValueGetters {
-		Segments[strings.ToLower(k)] = &segments{
+		Segments[k] = &segments{
 			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 		}
 	}
@@ -500,7 +500,6 @@ func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
 }
 
 func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {
-	tbl = strings.ToLower(tbl)
 	for _, seg := range v.VisibleSegments(tbl) {
 		if !(slot >= seg.from && slot < seg.to) {
 			continue
@@ -515,7 +514,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 
 	segName := snaptype.BeaconBlocks.FileName(0, fromSlot, toSlot)
 	// a little bit ugly.
-	segName = strings.ReplaceAll(segName, "beaconblocks", strings.ToLower(snapName))
+	segName = strings.ReplaceAll(segName, "beaconblocks", snapName)
 	f, _, _ := snaptype.ParseFileName(snapDir, segName)
 
 	compressCfg := seg.DefaultCfg
@@ -653,7 +652,7 @@ func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
 	view := s.View()
 	defer view.Close()
 
-	seg, ok := view.VisibleSegment(slot, strings.ToLower(tbl))
+	seg, ok := view.VisibleSegment(slot, tbl)
 	if !ok {
 		return nil, nil
 	}

From 810e0a7742aa36871e6c05b9540b79d5e95c76a5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:28:06 +0200
Subject: [PATCH 083/131] save

---
 cmd/capcli/cli.go | 123 +++++++++++++++++++++++-----------------------
 1 file changed, 61 insertions(+), 62 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 099832c210c..d588dc98588 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -589,71 +589,70 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 		return err
 	}
 	r.withPPROF.withProfile()
-	for {
-		hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
-		start := time.Now()
-		haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)
-		if err != nil {
+	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
+	start := time.Now()
+	haveState, err := hr.ReadHistoricalState(ctx, tx, r.CompareSlot)
+	if err != nil {
+		return err
+	}
+	endTime := time.Since(start)
+	hRoot, err := haveState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	log.Info("Got state", "slot", haveState.Slot(), "root", libcommon.Hash(hRoot), "elapsed", endTime)
+
+	if err := haveState.InitBeaconState(); err != nil {
+		return err
+	}
+
+	v := haveState.Version()
+	// encode and decode the state
+	enc, err := haveState.EncodeSSZ(nil)
+	if err != nil {
+		return err
+	}
+	haveState = state.New(beaconConfig)
+	if err := haveState.DecodeSSZ(enc, int(v)); err != nil {
+		return err
+	}
+	if r.Out != "" {
+		// create file
+		if err := os.WriteFile(r.Out, enc, 0644); err != nil {
 			return err
 		}
-		endTime := time.Since(start)
-		hRoot, err := haveState.HashSSZ()
-		if err != nil {
-			return err
+	}
+
+	hRoot, err = haveState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	if r.CompareFile == "" {
+		return nil
+	}
+	// Read the content of CompareFile in a []byte  without afero
+	rawBytes, err := os.ReadFile(r.CompareFile)
+	if err != nil {
+		return err
+	}
+	// Decode the []byte into a state
+	wantState := state.New(beaconConfig)
+	if err := wantState.DecodeSSZ(rawBytes, int(haveState.Version())); err != nil {
+		return err
+	}
+	wRoot, err := wantState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	if hRoot != wRoot {
+		for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
+			if haveState.BlockRoots().Get(i) != wantState.BlockRoots().Get(i) {
+				log.Info("block roots mismatch", "index", i, "have", haveState.BlockRoots().Get(i), "want", wantState.BlockRoots().Get(i))
+			}
 		}
-		log.Info("Got state", "slot", haveState.Slot(), "root", libcommon.Hash(hRoot), "elapsed", endTime)
-	}
-	// if err := haveState.InitBeaconState(); err != nil {
-	// 	return err
-	// }
-
-	// v := haveState.Version()
-	// // encode and decode the state
-	// enc, err := haveState.EncodeSSZ(nil)
-	// if err != nil {
-	// 	return err
-	// }
-	// haveState = state.New(beaconConfig)
-	// if err := haveState.DecodeSSZ(enc, int(v)); err != nil {
-	// 	return err
-	// }
-	// if r.Out != "" {
-	// 	// create file
-	// 	if err := os.WriteFile(r.Out, enc, 0644); err != nil {
-	// 		return err
-	// 	}
-	// }
-
-	// hRoot, err = haveState.HashSSZ()
-	// if err != nil {
-	// 	return err
-	// }
-	// if r.CompareFile == "" {
-	// 	return nil
-	// }
-	// // Read the content of CompareFile in a []byte  without afero
-	// rawBytes, err := os.ReadFile(r.CompareFile)
-	// if err != nil {
-	// 	return err
-	// }
-	// // Decode the []byte into a state
-	// wantState := state.New(beaconConfig)
-	// if err := wantState.DecodeSSZ(rawBytes, int(haveState.Version())); err != nil {
-	// 	return err
-	// }
-	// wRoot, err := wantState.HashSSZ()
-	// if err != nil {
-	// 	return err
-	// }
-	// if hRoot != wRoot {
-	// 	for i := 0; i < haveState.PreviousEpochParticipation().Length(); i++ {
-	// 		if haveState.BlockRoots().Get(i) != wantState.BlockRoots().Get(i) {
-	// 			log.Info("block roots mismatch", "index", i, "have", haveState.BlockRoots().Get(i), "want", wantState.BlockRoots().Get(i))
-	// 		}
-	// 	}
-	// 	return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot))
-	// }
-	// return nil
+		return fmt.Errorf("state mismatch: got %s, want %s", libcommon.Hash(hRoot), libcommon.Hash(wRoot))
+	}
+	return nil
 }
 
 type ArchiveSanitizer struct {

From 5470be10e5c24f5e040e985b1dbad80fb5616254 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:29:14 +0200
Subject: [PATCH 084/131] save

---
 cl/antiquary/state_antiquary.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index d8327bb0c42..6a108dc3bc1 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -419,11 +419,12 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		if err := s.stateSn.OpenFolder(); err != nil {
 			return err
 		}
+		blocksPerStatefulFile := snaptype.CaplinMergeLimit * 5
 		if err := s.stateSn.DumpCaplinState(
 			ctx,
 			s.stateSn.BlocksAvailable()+1,
 			s.currentState.Slot(),
-			snaptype.CaplinMergeLimit,
+			uint64(blocksPerStatefulFile),
 			s.sn.Salt,
 			s.dirs,
 			runtime.NumCPU(),

From 1df5851893c614b4bd489ecc93b6568571a22209 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:31:06 +0200
Subject: [PATCH 085/131] save

---
 cl/antiquary/state_antiquary.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 6a108dc3bc1..6c79a18838e 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -419,12 +419,16 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		if err := s.stateSn.OpenFolder(); err != nil {
 			return err
 		}
-		blocksPerStatefulFile := snaptype.CaplinMergeLimit * 5
+		blocksPerStatefulFile := uint64(snaptype.CaplinMergeLimit * 5)
+		from := s.stateSn.BlocksAvailable() + 1
+		if from+blocksPerStatefulFile+safetyMargin > s.currentState.Slot() {
+			return nil
+		}
 		if err := s.stateSn.DumpCaplinState(
 			ctx,
 			s.stateSn.BlocksAvailable()+1,
 			s.currentState.Slot(),
-			uint64(blocksPerStatefulFile),
+			blocksPerStatefulFile,
 			s.sn.Salt,
 			s.dirs,
 			runtime.NumCPU(),
@@ -433,6 +437,9 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		); err != nil {
 			return err
 		}
+		if err := s.stateSn.OpenFolder(); err != nil {
+			return err
+		}
 	}
 
 	return nil

From 2485742abce56e22a778e018e24d920463c7c3f0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:34:14 +0200
Subject: [PATCH 086/131] save

---
 cl/antiquary/state_antiquary.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 6c79a18838e..54a50f6c44c 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -432,7 +432,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 			s.sn.Salt,
 			s.dirs,
 			runtime.NumCPU(),
-			log.LvlDebug,
+			log.LvlInfo,
 			s.logger,
 		); err != nil {
 			return err

From 0fe1e645cfc96abb2d57f832af21c07ec80206c9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:38:28 +0200
Subject: [PATCH 087/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index c6901363dc4..10e286d91b7 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -467,6 +467,9 @@ type CaplinStateView struct {
 }
 
 func (s *CaplinStateSnapshots) View() *CaplinStateView {
+	if s == nil {
+		return nil
+	}
 	s.visibleSegmentsLock.RLock()
 	defer s.visibleSegmentsLock.RUnlock()
 
@@ -482,6 +485,9 @@ func (s *CaplinStateSnapshots) View() *CaplinStateView {
 }
 
 func (v *CaplinStateView) Close() {
+	if v == nil {
+		return
+	}
 	if v.closed {
 		return
 	}

From 48661596f344dd798e9c92f796c8ec94cf5a467f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:42:00 +0200
Subject: [PATCH 088/131] save

---
 cl/antiquary/state_antiquary.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 54a50f6c44c..ca0e290067b 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -424,10 +424,15 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		if from+blocksPerStatefulFile+safetyMargin > s.currentState.Slot() {
 			return nil
 		}
+		to := s.currentState.Slot()
+		if to < (safetyMargin + blocksPerStatefulFile) {
+			return nil
+		}
+		to = to - (safetyMargin + blocksPerStatefulFile)
 		if err := s.stateSn.DumpCaplinState(
 			ctx,
 			s.stateSn.BlocksAvailable()+1,
-			s.currentState.Slot(),
+			to,
 			blocksPerStatefulFile,
 			s.sn.Salt,
 			s.dirs,

From ed7d368bba666898fe871a34b376a24ae46f863d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:43:34 +0200
Subject: [PATCH 089/131] save

---
 cl/antiquary/state_antiquary.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index ca0e290067b..48f050bc7e0 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -429,6 +429,9 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 			return nil
 		}
 		to = to - (safetyMargin + blocksPerStatefulFile)
+		if from > to {
+			return nil
+		}
 		if err := s.stateSn.DumpCaplinState(
 			ctx,
 			s.stateSn.BlocksAvailable()+1,

From 07c307531a1afabbc204a24bae56d090f9f78e4b Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:43:40 +0200
Subject: [PATCH 090/131] save

---
 cl/antiquary/state_antiquary.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 48f050bc7e0..a8ceada9a39 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -429,7 +429,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 			return nil
 		}
 		to = to - (safetyMargin + blocksPerStatefulFile)
-		if from > to {
+		if from >= to {
 			return nil
 		}
 		if err := s.stateSn.DumpCaplinState(

From 1052ee67081dfddf7b3d10f250714ffa39a01d1a Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:02:24 +0100
Subject: [PATCH 091/131] save

---
 cl/beacon/handler/utils_test.go                            | 5 +++--
 cl/beacon/handler/validator_test.go                        | 1 +
 .../historical_states_reader/historical_states_reader.go   | 7 ++++++-
 .../historical_states_reader_test.go                       | 4 ++--
 cl/sentinel/sentinel_requests_test.go                      | 2 +-
 5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/cl/beacon/handler/utils_test.go b/cl/beacon/handler/utils_test.go
index f480f849b2f..a1c0e0f8ddf 100644
--- a/cl/beacon/handler/utils_test.go
+++ b/cl/beacon/handler/utils_test.go
@@ -78,10 +78,10 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &bcfg, datadir.New("/tmp"), nil, db, nil, reader, logger, true, true, false, false, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &bcfg, datadir.New("/tmp"), nil, db, nil, nil, reader, logger, true, true, false, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	// historical states reader below
-	statesReader := historical_states_reader.NewHistoricalStatesReader(&bcfg, reader, vt, preState)
+	statesReader := historical_states_reader.NewHistoricalStatesReader(&bcfg, reader, vt, preState, nil)
 	opPool = pool.NewOperationsPool(&bcfg)
 	fcu.Pool = opPool
 
@@ -176,6 +176,7 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
 		proposerSlashingService,
 		nil,
 		mockValidatorMonitor,
+		nil,
 	) // TODO: add tests
 	h.Init()
 	return
diff --git a/cl/beacon/handler/validator_test.go b/cl/beacon/handler/validator_test.go
index 533e45b3bef..5a1979e58c3 100644
--- a/cl/beacon/handler/validator_test.go
+++ b/cl/beacon/handler/validator_test.go
@@ -75,6 +75,7 @@ func (t *validatorTestSuite) SetupTest() {
 		nil,
 		nil,
 		nil,
+		nil,
 	)
 	t.gomockCtrl = gomockCtrl
 }
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 2ccf35e5c98..9e05e75600d 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -85,7 +85,12 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	if err != nil {
 		return nil, err
 	}
-	latestProcessedState = max(latestProcessedState, r.stateSn.BlocksAvailable())
+
+	var blocksAvailableInSnapshots uint64
+	if r.stateSn != nil {
+		blocksAvailableInSnapshots = r.stateSn.BlocksAvailable()
+	}
+	latestProcessedState = max(latestProcessedState, blocksAvailableInSnapshots))
 
 	// If this happens, we need to update our static tables
 	if slot > latestProcessedState || slot > r.validatorTable.Slot() {
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go
index 38151494504..290239143bd 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader_test.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader_test.go
@@ -41,7 +41,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, false, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	// Now lets test it against the reader
 	tx, err := db.BeginRw(ctx)
@@ -50,7 +50,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	vt = state_accessors.NewStaticValidatorTable()
 	require.NoError(t, state_accessors.ReadValidatorsTable(tx, vt))
-	hr := historical_states_reader.NewHistoricalStatesReader(&clparams.MainnetBeaconConfig, reader, vt, preState)
+	hr := historical_states_reader.NewHistoricalStatesReader(&clparams.MainnetBeaconConfig, reader, vt, preState, nil)
 	s, err := hr.ReadHistoricalState(ctx, tx, blocks[len(blocks)-1].Block.Slot)
 	require.NoError(t, err)
 
diff --git a/cl/sentinel/sentinel_requests_test.go b/cl/sentinel/sentinel_requests_test.go
index cfd0fa65cd4..3fe6d03050d 100644
--- a/cl/sentinel/sentinel_requests_test.go
+++ b/cl/sentinel/sentinel_requests_test.go
@@ -55,7 +55,7 @@ func loadChain(t *testing.T) (db kv.RwDB, blocks []*cltypes.SignedBeaconBlock, f
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, false, false, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, log.New(), true, true, false, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	return
 }

From ac4e98c71e6c11b94497830d0cb290aa9663c9d4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:08:15 +0100
Subject: [PATCH 092/131] save

---
 .../state/historical_states_reader/historical_states_reader.go  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 9e05e75600d..894370a7235 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -90,7 +90,7 @@ func (r *HistoricalStatesReader) ReadHistoricalState(ctx context.Context, tx kv.
 	if r.stateSn != nil {
 		blocksAvailableInSnapshots = r.stateSn.BlocksAvailable()
 	}
-	latestProcessedState = max(latestProcessedState, blocksAvailableInSnapshots))
+	latestProcessedState = max(latestProcessedState, blocksAvailableInSnapshots)
 
 	// If this happens, we need to update our static tables
 	if slot > latestProcessedState || slot > r.validatorTable.Slot() {

From a4de21e796c4bc1410febf039f93be9b8af761bc Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:23:53 +0100
Subject: [PATCH 093/131] save

---
 cl/antiquary/state_antiquary.go          | 61 ++++++++++++++++++++++++
 cl/persistence/state/validator_events.go |  4 ++
 2 files changed, 65 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index a8ceada9a39..65a39b429d8 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -121,9 +121,70 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	return
 }
 
+func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
+	if s.stateSn == nil || s.stateSn.BlocksAvailable() <= s.validatorsTable.Slot() {
+		return nil
+	}
+	if err := s.stateSn.OpenFolder(); err != nil {
+		return err
+	}
+	blocksAvaiable := s.stateSn.BlocksAvailable()
+	stateSnRoTx := s.stateSn.View()
+	defer stateSnRoTx.Close()
+
+	start := time.Now()
+	lastLog := time.Now()
+	for slot := s.validatorsTable.Slot() + 1; slot <= blocksAvaiable; slot++ {
+		if slot%100_000 == 0 {
+			s.logger.Info("[Antiquary] Filling static validators table", "slot", slot, "elapsed", time.Since(lastLog))
+			lastLog = time.Now()
+		}
+		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
+		if !ok {
+			return fmt.Errorf("segment not found for slot %d", slot)
+		}
+		buf, err := seg.Get(slot)
+		if err != nil {
+			return err
+		}
+		if len(buf) == 0 {
+			continue
+		}
+		event := state_accessors.NewStateEventsFromBytes(buf)
+		state_accessors.ReplayEvents(
+			func(validatorIndex uint64, validator solid.Validator) error {
+				return s.validatorsTable.AddValidator(validator, validatorIndex, slot)
+			},
+			func(validatorIndex uint64, exitEpoch uint64) error {
+				return s.validatorsTable.AddExitEpoch(validatorIndex, slot, exitEpoch)
+			},
+			func(validatorIndex uint64, withdrawableEpoch uint64) error {
+				return s.validatorsTable.AddWithdrawableEpoch(validatorIndex, slot, withdrawableEpoch)
+			},
+			func(validatorIndex uint64, withdrawalCredentials libcommon.Hash) error {
+				return s.validatorsTable.AddWithdrawalCredentials(validatorIndex, slot, withdrawalCredentials)
+			},
+			func(validatorIndex uint64, activationEpoch uint64) error {
+				return s.validatorsTable.AddActivationEpoch(validatorIndex, slot, activationEpoch)
+			},
+			func(validatorIndex uint64, activationEligibilityEpoch uint64) error {
+				return s.validatorsTable.AddActivationEligibility(validatorIndex, slot, activationEligibilityEpoch)
+			},
+			func(validatorIndex uint64, slashed bool) error {
+				return s.validatorsTable.AddSlashed(validatorIndex, slot, slashed)
+			},
+			event,
+		)
+	}
+	s.logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
+	return nil
+}
+
 func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	var tx kv.Tx
 
+	// Check if you need to fill the static validators table
+
 	tx, err := s.mainDB.BeginRo(ctx)
 	if err != nil {
 		return err
diff --git a/cl/persistence/state/validator_events.go b/cl/persistence/state/validator_events.go
index e6a1c9e7e2a..b6c8e83d9ed 100644
--- a/cl/persistence/state/validator_events.go
+++ b/cl/persistence/state/validator_events.go
@@ -50,6 +50,10 @@ func NewStateEvents() *StateEvents {
 	return &StateEvents{}
 }
 
+func NewStateEventsFromBytes(buf []byte) *StateEvents {
+	return &StateEvents{buf: libcommon.Copy(buf)}
+}
+
 func (se *StateEvents) AddValidator(validatorIndex uint64, validator solid.Validator) {
 	se.mu.Lock()
 	defer se.mu.Unlock()

From bfd791afb28e916da3ea0e31f23a8be243b2195e Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:24:07 +0100
Subject: [PATCH 094/131] save

---
 cl/antiquary/state_antiquary.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 65a39b429d8..8743076ca55 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -184,6 +184,9 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	var tx kv.Tx
 
 	// Check if you need to fill the static validators table
+	if err := s.fillStaticValidatorsTable(ctx); err != nil {
+		return err
+	}
 
 	tx, err := s.mainDB.BeginRo(ctx)
 	if err != nil {

From 0766013d54d7f8897f292d0ec34f6662ae3a9025 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 02:33:27 +0100
Subject: [PATCH 095/131] save

---
 cl/antiquary/state_antiquary.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 8743076ca55..ef9ad910464 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -113,6 +113,9 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	if err != nil {
 		return
 	}
+	if s.stateSn != nil {
+		progress = max(progress, s.stateSn.BlocksAvailable())
+	}
 
 	finalized, err = beacon_indicies.ReadHighestFinalized(tx)
 	if err != nil {

From b07c55cb14984ff103f4dc3d076197ceb37d3e1a Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 03:07:01 +0100
Subject: [PATCH 096/131] save

---
 cl/antiquary/state_antiquary.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index ef9ad910464..29abd02fc8f 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -543,6 +543,9 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 	if err != nil {
 		return err
 	}
+	if s.stateSn != nil {
+		targetSlot = max(targetSlot, s.stateSn.BlocksAvailable())
+	}
 	// We want to backoff by some slots until we get a correct state from DB.
 	// we start from 10 * clparams.SlotsPerDump.
 	backoffStrides := uint64(10)

From 26c7d9dad66e4a421b23b080350bd7da41d904f0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 03:09:33 +0100
Subject: [PATCH 097/131] save

---
 cl/antiquary/state_antiquary.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 29abd02fc8f..9def705ad60 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -178,6 +178,7 @@ func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
 			},
 			event,
 		)
+		s.validatorsTable.SetSlot(slot)
 	}
 	s.logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
 	return nil

From b88149664424c904890ea2f0caf1d7bca420f4ed Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 03:23:27 +0100
Subject: [PATCH 098/131] save

---
 cl/antiquary/state_antiquary.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 9def705ad60..01fc3316086 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -573,6 +573,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 		if err != nil {
 			return fmt.Errorf("failed to read historical state at slot %d: %w", attempt, err)
 		}
+		fmt.Println("currentState", s.currentState, attempt)
 		if s.currentState == nil {
 			log.Warn("historical state not found, backoff more and try again", "slot", attempt)
 			backoffStep += backoffStrides

From 70c0538f6e055715e220924935da6e93f630fdc3 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 03:28:16 +0100
Subject: [PATCH 099/131] save

---
 cl/antiquary/state_antiquary.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 01fc3316086..2e13e803acf 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -134,14 +134,16 @@ func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
 	blocksAvaiable := s.stateSn.BlocksAvailable()
 	stateSnRoTx := s.stateSn.View()
 	defer stateSnRoTx.Close()
+	if s.genesisState == nil {
+		return fmt.Errorf("genesis state is nil")
+	}
+	startSlot := s.validatorsTable.Slot() + 1
+	if startSlot == 1 {
+		startSlot = 0
+	}
 
 	start := time.Now()
-	lastLog := time.Now()
 	for slot := s.validatorsTable.Slot() + 1; slot <= blocksAvaiable; slot++ {
-		if slot%100_000 == 0 {
-			s.logger.Info("[Antiquary] Filling static validators table", "slot", slot, "elapsed", time.Since(lastLog))
-			lastLog = time.Now()
-		}
 		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
 		if !ok {
 			return fmt.Errorf("segment not found for slot %d", slot)

From 2953bbfbac5613c77ce1375eca2f627652119dcd Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 16:55:34 +0100
Subject: [PATCH 100/131] save

---
 cl/antiquary/state_antiquary.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 2e13e803acf..8202eb1db5c 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -143,7 +143,7 @@ func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
 	}
 
 	start := time.Now()
-	for slot := s.validatorsTable.Slot() + 1; slot <= blocksAvaiable; slot++ {
+	for slot := startSlot; slot <= blocksAvaiable; slot++ {
 		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
 		if !ok {
 			return fmt.Errorf("segment not found for slot %d", slot)

From 37838b0979cb9f3b5d30af80a92e817d958e648f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:02:41 +0100
Subject: [PATCH 101/131] save

---
 cl/antiquary/state_antiquary.go | 37 ++++++++++++++++-----------------
 cmd/capcli/cli.go               |  4 ++++
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 8202eb1db5c..c0ccff1571d 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -44,6 +44,7 @@ import (
 	"github.com/erigontech/erigon/cl/phase1/core/state/raw"
 	"github.com/erigontech/erigon/cl/transition"
 	"github.com/erigontech/erigon/cl/transition/impl/eth2"
+	"github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 )
 
 // pool for buffers
@@ -124,20 +125,18 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	return
 }
 
-func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
-	if s.stateSn == nil || s.stateSn.BlocksAvailable() <= s.validatorsTable.Slot() {
+func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, stateSn *freezeblocks.CaplinStateSnapshots, validatorsTable *state_accessors.StaticValidatorTable) error {
+	if stateSn == nil || stateSn.BlocksAvailable() <= validatorsTable.Slot() {
 		return nil
 	}
-	if err := s.stateSn.OpenFolder(); err != nil {
+	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
-	blocksAvaiable := s.stateSn.BlocksAvailable()
-	stateSnRoTx := s.stateSn.View()
+	blocksAvaiable := stateSn.BlocksAvailable()
+	stateSnRoTx := stateSn.View()
 	defer stateSnRoTx.Close()
-	if s.genesisState == nil {
-		return fmt.Errorf("genesis state is nil")
-	}
-	startSlot := s.validatorsTable.Slot() + 1
+
+	startSlot := validatorsTable.Slot() + 1
 	if startSlot == 1 {
 		startSlot = 0
 	}
@@ -158,31 +157,31 @@ func (s *Antiquary) fillStaticValidatorsTable(ctx context.Context) error {
 		event := state_accessors.NewStateEventsFromBytes(buf)
 		state_accessors.ReplayEvents(
 			func(validatorIndex uint64, validator solid.Validator) error {
-				return s.validatorsTable.AddValidator(validator, validatorIndex, slot)
+				return validatorsTable.AddValidator(validator, validatorIndex, slot)
 			},
 			func(validatorIndex uint64, exitEpoch uint64) error {
-				return s.validatorsTable.AddExitEpoch(validatorIndex, slot, exitEpoch)
+				return validatorsTable.AddExitEpoch(validatorIndex, slot, exitEpoch)
 			},
 			func(validatorIndex uint64, withdrawableEpoch uint64) error {
-				return s.validatorsTable.AddWithdrawableEpoch(validatorIndex, slot, withdrawableEpoch)
+				return validatorsTable.AddWithdrawableEpoch(validatorIndex, slot, withdrawableEpoch)
 			},
 			func(validatorIndex uint64, withdrawalCredentials libcommon.Hash) error {
-				return s.validatorsTable.AddWithdrawalCredentials(validatorIndex, slot, withdrawalCredentials)
+				return validatorsTable.AddWithdrawalCredentials(validatorIndex, slot, withdrawalCredentials)
 			},
 			func(validatorIndex uint64, activationEpoch uint64) error {
-				return s.validatorsTable.AddActivationEpoch(validatorIndex, slot, activationEpoch)
+				return validatorsTable.AddActivationEpoch(validatorIndex, slot, activationEpoch)
 			},
 			func(validatorIndex uint64, activationEligibilityEpoch uint64) error {
-				return s.validatorsTable.AddActivationEligibility(validatorIndex, slot, activationEligibilityEpoch)
+				return validatorsTable.AddActivationEligibility(validatorIndex, slot, activationEligibilityEpoch)
 			},
 			func(validatorIndex uint64, slashed bool) error {
-				return s.validatorsTable.AddSlashed(validatorIndex, slot, slashed)
+				return validatorsTable.AddSlashed(validatorIndex, slot, slashed)
 			},
 			event,
 		)
-		s.validatorsTable.SetSlot(slot)
+		validatorsTable.SetSlot(slot)
 	}
-	s.logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
+	logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
 	return nil
 }
 
@@ -190,7 +189,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	var tx kv.Tx
 
 	// Check if you need to fill the static validators table
-	if err := s.fillStaticValidatorsTable(ctx); err != nil {
+	if err := FillStaticValidatorsTableIfNeeded(ctx, s.logger, s.stateSn, s.validatorsTable); err != nil {
 		return err
 	}
 
diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index d588dc98588..23d2969fdb7 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -588,6 +588,10 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
+	if err := antiquary.FillStaticValidatorsTableIfNeeded(ctx, log.Root(), stateSn, vt); err != nil {
+		return err
+	}
+	fmt.Println(vt.ActivationEpoch(0, 1))
 	r.withPPROF.withProfile()
 	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
 	start := time.Now()

From a1259e12d368674d438511289b5a28c4d5690d40 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:04:42 +0100
Subject: [PATCH 102/131] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 23d2969fdb7..a8fe0d1bebd 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -591,7 +591,7 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 	if err := antiquary.FillStaticValidatorsTableIfNeeded(ctx, log.Root(), stateSn, vt); err != nil {
 		return err
 	}
-	fmt.Println(vt.ActivationEpoch(0, 1))
+	fmt.Println(vt.WithdrawableEpoch(0, 1))
 	r.withPPROF.withProfile()
 	hr := historical_states_reader.NewHistoricalStatesReader(beaconConfig, snr, vt, gSpot, stateSn)
 	start := time.Now()

From 106f4e500db8f67771d5cc9b285da4561942d093 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:05:41 +0100
Subject: [PATCH 103/131] save

---
 cl/phase1/core/state/raw/hashing.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index a4390593baf..79a286cf01f 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,6 +17,7 @@
 package raw
 
 import (
+	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -31,9 +32,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	// for i := 0; i < len(b.leaves); i += 32 {
-	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	// }
+	for i := 0; i < len(b.leaves); i += 32 {
+		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	}
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return

From a13ef79d5650c55c5a05bad924b15d9d20d2037d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:11:07 +0100
Subject: [PATCH 104/131] save

---
 cl/persistence/state/state_accessors.go |  2 ++
 holy.txt                                | 43 ++++++++++++++++++++++---
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 6d4ca4d6c10..68e71ee38f5 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -19,6 +19,7 @@ package state_accessors
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -41,6 +42,7 @@ func GetValFnTxAndSnapshot(tx kv.Tx, snapshotRoTx *freezeblocks.CaplinStateView)
 				return segment.Get(slot)
 			}
 		}
+		fmt.Println("mis", table)
 		return tx.GetOne(table, key)
 	}
 }
diff --git a/holy.txt b/holy.txt
index 921062010cf..a1e9dee8119 100644
--- a/holy.txt
+++ b/holy.txt
@@ -3,18 +3,51 @@
 2 0x41420f0000000000000000000000000000000000000000000000000000000000                                                                                                                                                  
 3 0x30680b18ada5ceaeb222d0d15bd7058fbc4e1e5e3c002d504adee82cc013a517                                                                                                                                                  
 4 0x1158a7769cb0dfc3d015aa03bd76760d57bab8ad2fb77885a07917f1754194cf                                                                                                                                                  
-5 0xfe1acfde1678da01248a9825ec08d4f3ee73615c1263095077ee23063d1819cb                                                                                                                                                  
-6 0x0ead07fe6e6026ee73f966aceffaaeff6b50d6df923e5230a9cc33023d52adbe                                                                                                                                                  
+5 0x986c1ffc2b1ecc1e36e4cb94ad932fed8ff8d33e8a2ac6729e905f1d503ac9cd                                                                                                                                                  
+6 0x9d376a65dfa13037b5fe1c2066dfb7764a8e026e62d2fca64d15828a8123fafa                                                                                                                                                  
 7 0xb6049c0ff9220d02083c30474400cbcb358a6d6804082757553468badccd8a48                                                                                                                                                  
 8 0xa4007911445daf00197d2cca907cebf0529773561b702a2136decb52edc87348                                                                                                                                                  
-9 0xc231c219f35945a38dbab773bf4f32c9080c59f9e6284fbb92ffe02c3318165d
+9 0xec2cf9d12a2e597caa03cb8c9ffada78b83fe44c940f8737b076af563c289a67
 10 0x9301000000000000000000000000000000000000000000000000000000000000
 11 0xed774cba995d2b5cd55a05ac5126d4fb24a1dbf7523ebdc0b1dc8f7f02de30d6
 12 0xe7c27b3e6aaccb0c4f42eda4ffadd5f92fc28edfa5ef573f28ab5495f669282a
-13 0xe65a9db1e18c1435832c978ca666c5502dab9b85bf1258623e8a2e96e986e78f
+13 0xc64d19f5ea9664a3f7a8bae036771b68a5d22b646a3f16c34e4c29af0213ecb0
+14 0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220
+15 0x8fe5cf794229f584ba98f5f351b97617f0e3854d6a08e1fb7bc3ee75d67b2469
+16 0x8fe5cf794229f584ba98f5f351b97617f0e3854d6a08e1fb7bc3ee75d67b2469
+17 0x0f00000000000000000000000000000000000000000000000000000000000000
+18 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
+19 0xe74aa97c38fea5a58af5514cd78e2180f0ba62f0709676c62d6e9af62fff77d0
+20 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
+21 0x3d4be5d019ba15ea3ef304a83b8a067f2e79f46a3fac8069306a6c814a0a35eb
+22 0x2961343e46fbbea5ef5373a5db989b5e2ed19baef61d9f26ee6ac63a9023862b
+23 0x3843792806b2d08dd125002fd3e68d117c7860e4c863f7f60d59625a6f220d48
+24 0x69c41e47f10b14f920a3851f8a4f71d19f83a8d1e28204efccccd644747141be
+25 0x0000000000000000000000000000000000000000000000000000000000000000
+26 0x0000000000000000000000000000000000000000000000000000000000000000
+27 0x0000000000000000000000000000000000000000000000000000000000000000
+28 0x0000000000000000000000000000000000000000000000000000000000000000
+29 0x0000000000000000000000000000000000000000000000000000000000000000
+30 0x0000000000000000000000000000000000000000000000000000000000000000
+31 0x0000000000000000000000000000000000000000000000000000000000000000
+
+0 0x607db06200000000000000000000000000000000000000000000000000000000                                                                                                                                                  
+1 0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078                                                                                                                                                  
+2 0x41420f0000000000000000000000000000000000000000000000000000000000                                                                                                                                                  
+3 0x30680b18ada5ceaeb222d0d15bd7058fbc4e1e5e3c002d504adee82cc013a517                                                                                                                                                  
+4 0x1158a7769cb0dfc3d015aa03bd76760d57bab8ad2fb77885a07917f1754194cf                                                                                                                                                  
+5 0x986c1ffc2b1ecc1e36e4cb94ad932fed8ff8d33e8a2ac6729e905f1d503ac9cd                                                                                                                                                  
+6 0x9d376a65dfa13037b5fe1c2066dfb7764a8e026e62d2fca64d15828a8123fafa                                                                                                                                                  
+7 0xb6049c0ff9220d02083c30474400cbcb358a6d6804082757553468badccd8a48                                                                                                                                                  
+8 0xa4007911445daf00197d2cca907cebf0529773561b702a2136decb52edc87348                                                                                                                                                  
+9 0xec2cf9d12a2e597caa03cb8c9ffada78b83fe44c940f8737b076af563c289a67
+10 0x9301000000000000000000000000000000000000000000000000000000000000
+11 0xed774cba995d2b5cd55a05ac5126d4fb24a1dbf7523ebdc0b1dc8f7f02de30d6
+12 0xe7c27b3e6aaccb0c4f42eda4ffadd5f92fc28edfa5ef573f28ab5495f669282a
+13 0xc64d19f5ea9664a3f7a8bae036771b68a5d22b646a3f16c34e4c29af0213ecb0
 14 0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220
 15 0xe15e05e8cb173b5d2a4bc5acea5aab50ea2e616fcc41bee9833377ce40135897
-16 0xe9eddec5525f068001fdff036039e7946248a1d80ce8f6b40a1e79d01303519b
+16 0xe1550113957228a811660a70c48bccbf1acc5f511aad5beff415cb23c5a6b2b7
 17 0x0f00000000000000000000000000000000000000000000000000000000000000
 18 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
 19 0xe74aa97c38fea5a58af5514cd78e2180f0ba62f0709676c62d6e9af62fff77d0

From e8cc73323840927ab39a44bac61fc4b0b401eadb Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:16:10 +0100
Subject: [PATCH 105/131] save

---
 .../state/historical_states_reader/attesting_indicies.go  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/attesting_indicies.go b/cl/persistence/state/historical_states_reader/attesting_indicies.go
index 14001ba3808..ea7536350c2 100644
--- a/cl/persistence/state/historical_states_reader/attesting_indicies.go
+++ b/cl/persistence/state/historical_states_reader/attesting_indicies.go
@@ -132,7 +132,7 @@ func committeeCount(cfg *clparams.BeaconChainConfig, epoch uint64, idxs []uint64
 	return committeCount
 }
 
-func (r *HistoricalStatesReader) readHistoricalBlockRoot(tx kv.Tx, slot, index uint64) (libcommon.Hash, error) {
+func (r *HistoricalStatesReader) readHistoricalBlockRoot(kvGetter state_accessors.GetValFn, slot, index uint64) (libcommon.Hash, error) {
 	slotSubIndex := slot % r.cfg.SlotsPerHistoricalRoot
 	needFromGenesis := true
 
@@ -152,7 +152,7 @@ func (r *HistoricalStatesReader) readHistoricalBlockRoot(tx kv.Tx, slot, index u
 	if needFromGenesis {
 		return r.genesisState.GetBlockRootAtSlot(slot)
 	}
-	br, err := tx.GetOne(kv.BlockRoot, base_encoding.Encode64ToBytes4(slotLookup))
+	br, err := kvGetter(kv.BlockRoot, base_encoding.Encode64ToBytes4(slotLookup))
 	if err != nil {
 		return libcommon.Hash{}, err
 	}
@@ -187,13 +187,13 @@ func (r *HistoricalStatesReader) getAttestationParticipationFlagIndicies(tx kv.T
 		return nil, errors.New("GetAttestationParticipationFlagIndicies: source does not match.")
 	}
 	i := (data.Target.Epoch * r.cfg.SlotsPerEpoch) % r.cfg.SlotsPerHistoricalRoot
-	targetRoot, err := r.readHistoricalBlockRoot(tx, stateSlot, i)
+	targetRoot, err := r.readHistoricalBlockRoot(getter, stateSlot, i)
 	if err != nil {
 		return nil, err
 	}
 
 	i = data.Slot % r.cfg.SlotsPerHistoricalRoot
-	headRoot, err := r.readHistoricalBlockRoot(tx, stateSlot, i)
+	headRoot, err := r.readHistoricalBlockRoot(getter, stateSlot, i)
 	if err != nil {
 		return nil, err
 	}

From 41470647f163aadfee91eb70ef542ae73f8554d5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:17:23 +0100
Subject: [PATCH 106/131] save

---
 cl/phase1/core/state/raw/hashing.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go
index 79a286cf01f..a4390593baf 100644
--- a/cl/phase1/core/state/raw/hashing.go
+++ b/cl/phase1/core/state/raw/hashing.go
@@ -17,7 +17,6 @@
 package raw
 
 import (
-	"fmt"
 	"sync"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -32,9 +31,9 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	if err = b.computeDirtyLeaves(); err != nil {
 		return [32]byte{}, err
 	}
-	for i := 0; i < len(b.leaves); i += 32 {
-		fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
-	}
+	// for i := 0; i < len(b.leaves); i += 32 {
+	// 	fmt.Println(i/32, libcommon.BytesToHash(b.leaves[i:i+32]))
+	// }
 	// Pad to 32 of length
 	err = merkle_tree.MerkleRootFromFlatLeaves(b.leaves, out[:])
 	return

From 032aed13b4dd8d28d097aa6041f56ebf6c913f51 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:17:58 +0100
Subject: [PATCH 107/131] save

---
 cl/persistence/state/state_accessors.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/cl/persistence/state/state_accessors.go b/cl/persistence/state/state_accessors.go
index 68e71ee38f5..6d4ca4d6c10 100644
--- a/cl/persistence/state/state_accessors.go
+++ b/cl/persistence/state/state_accessors.go
@@ -19,7 +19,6 @@ package state_accessors
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/cltypes"
@@ -42,7 +41,6 @@ func GetValFnTxAndSnapshot(tx kv.Tx, snapshotRoTx *freezeblocks.CaplinStateView)
 				return segment.Get(slot)
 			}
 		}
-		fmt.Println("mis", table)
 		return tx.GetOne(table, key)
 	}
 }

From 4df47c78461c885e68985c7105e5a18505b2e855 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 17:45:45 +0100
Subject: [PATCH 108/131] save

---
 cl/antiquary/state_antiquary.go | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index c0ccff1571d..6b73201e33b 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -126,7 +126,7 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 }
 
 func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, stateSn *freezeblocks.CaplinStateSnapshots, validatorsTable *state_accessors.StaticValidatorTable) error {
-	if stateSn == nil || stateSn.BlocksAvailable() <= validatorsTable.Slot() {
+	if stateSn == nil || validatorsTable.Slot() != 0 {
 		return nil
 	}
 	if err := stateSn.OpenFolder(); err != nil {
@@ -136,13 +136,8 @@ func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, s
 	stateSnRoTx := stateSn.View()
 	defer stateSnRoTx.Close()
 
-	startSlot := validatorsTable.Slot() + 1
-	if startSlot == 1 {
-		startSlot = 0
-	}
-
 	start := time.Now()
-	for slot := startSlot; slot <= blocksAvaiable; slot++ {
+	for slot := uint64(0); slot <= stateSn.BlocksAvailable(); slot++ {
 		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
 		if !ok {
 			return fmt.Errorf("segment not found for slot %d", slot)

From 20f055e5ae5bb1b969f987f4676cce856baffc4d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 18:13:01 +0100
Subject: [PATCH 109/131] save

---
 cl/antiquary/state_antiquary.go | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 6b73201e33b..24d04604f81 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -125,12 +125,12 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	return
 }
 
-func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, stateSn *freezeblocks.CaplinStateSnapshots, validatorsTable *state_accessors.StaticValidatorTable) error {
+func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, stateSn *freezeblocks.CaplinStateSnapshots, validatorsTable *state_accessors.StaticValidatorTable) (bool, error) {
 	if stateSn == nil || validatorsTable.Slot() != 0 {
-		return nil
+		return false, nil
 	}
 	if err := stateSn.OpenFolder(); err != nil {
-		return err
+		return false, err
 	}
 	blocksAvaiable := stateSn.BlocksAvailable()
 	stateSnRoTx := stateSn.View()
@@ -140,11 +140,11 @@ func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, s
 	for slot := uint64(0); slot <= stateSn.BlocksAvailable(); slot++ {
 		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
 		if !ok {
-			return fmt.Errorf("segment not found for slot %d", slot)
+			return false, fmt.Errorf("segment not found for slot %d", slot)
 		}
 		buf, err := seg.Get(slot)
 		if err != nil {
-			return err
+			return false, err
 		}
 		if len(buf) == 0 {
 			continue
@@ -177,14 +177,14 @@ func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, s
 		validatorsTable.SetSlot(slot)
 	}
 	logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
-	return nil
+	return true, nil
 }
 
 func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
-	var tx kv.Tx
 
 	// Check if you need to fill the static validators table
-	if err := FillStaticValidatorsTableIfNeeded(ctx, s.logger, s.stateSn, s.validatorsTable); err != nil {
+	refilledStaticValidators, err := FillStaticValidatorsTableIfNeeded(ctx, s.logger, s.stateSn, s.validatorsTable)
+	if err != nil {
 		return err
 	}
 
@@ -197,6 +197,13 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	// maps which validators changes
 	var changedValidators sync.Map
 
+	if refilledStaticValidators {
+		s.validatorsTable.ForEach(func(validatorIndex uint64, validator *state_accessors.StaticValidator) bool {
+			changedValidators.Store(validatorIndex, struct{}{})
+			return true
+		})
+	}
+
 	stateAntiquaryCollector := newBeaconStatesCollector(s.cfg, s.dirs.Tmp, s.logger)
 	defer stateAntiquaryCollector.close()
 

From 8a8726ecc777f6d2f53ef261d7f58de46dca5d86 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 18:43:47 +0100
Subject: [PATCH 110/131] save

---
 .../historical_states_reader/historical_states_reader.go     | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 894370a7235..65613ccb419 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -787,10 +787,7 @@ func (r *HistoricalStatesReader) ReadParticipations(tx kv.Tx, kvGetter state_acc
 	if err != nil {
 		return nil, nil, err
 	}
-	// trigger the cache for shuffled sets in parallel
-	if err := r.tryCachingEpochsInParallell(tx, kvGetter, [][]uint64{currentActiveIndicies, previousActiveIndicies}, []uint64{epoch, prevEpoch}); err != nil {
-		return nil, nil, err
-	}
+
 	// Read the previous idxs
 	for i := beginSlot; i <= slot; i++ {
 		// Read the block

From d8ed11b14d0a25ece4667080d9cdc63145a49ac8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 18:45:17 +0100
Subject: [PATCH 111/131] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a8fe0d1bebd..948503f3253 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -588,7 +588,7 @@ func (r *RetrieveHistoricalState) Run(ctx *Context) error {
 	if err := stateSn.OpenFolder(); err != nil {
 		return err
 	}
-	if err := antiquary.FillStaticValidatorsTableIfNeeded(ctx, log.Root(), stateSn, vt); err != nil {
+	if _, err := antiquary.FillStaticValidatorsTableIfNeeded(ctx, log.Root(), stateSn, vt); err != nil {
 		return err
 	}
 	fmt.Println(vt.WithdrawableEpoch(0, 1))

From a99e1c43c6b0bbe4269a1f65c827f003dcb58d2e Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 18:48:20 +0100
Subject: [PATCH 112/131] save

---
 .../attesting_indicies.go                      | 18 +++---------------
 .../historical_states_reader.go                |  3 ---
 2 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/attesting_indicies.go b/cl/persistence/state/historical_states_reader/attesting_indicies.go
index ea7536350c2..9da5eb8be79 100644
--- a/cl/persistence/state/historical_states_reader/attesting_indicies.go
+++ b/cl/persistence/state/historical_states_reader/attesting_indicies.go
@@ -19,13 +19,11 @@ package historical_states_reader
 import (
 	"errors"
 	"fmt"
-	"time"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon/cl/clparams"
 	"github.com/erigontech/erigon/cl/cltypes/solid"
-	"github.com/erigontech/erigon/cl/monitor/shuffling_metrics"
 	"github.com/erigontech/erigon/cl/persistence/base_encoding"
 	state_accessors "github.com/erigontech/erigon/cl/persistence/state"
 	"github.com/erigontech/erigon/cl/phase1/core/state/shuffling"
@@ -104,19 +102,9 @@ func (r *HistoricalStatesReader) ComputeCommittee(mix libcommon.Hash, indicies [
 	start := (lenIndicies * index) / count
 	end := (lenIndicies * (index + 1)) / count
 	var shuffledIndicies []uint64
-	epoch := slot / cfg.SlotsPerEpoch
-	/*
-	   mixPosition := (epoch + cfg.EpochsPerHistoricalVector - cfg.MinSeedLookahead - 1) % cfg.EpochsPerHistoricalVector
-	*/
-	if shuffledIndicesInterface, ok := r.shuffledSetsCache.Get(epoch); ok {
-		shuffledIndicies = shuffledIndicesInterface
-	} else {
-		shuffledIndicies = make([]uint64, lenIndicies)
-		start := time.Now()
-		shuffledIndicies = shuffling.ComputeShuffledIndicies(cfg, mix, shuffledIndicies, indicies, slot)
-		shuffling_metrics.ObserveComputeShuffledIndiciesTime(start)
-		r.shuffledSetsCache.Add(epoch, shuffledIndicies)
-	}
+
+	shuffledIndicies = make([]uint64, lenIndicies)
+	shuffledIndicies = shuffling.ComputeShuffledIndicies(cfg, mix, shuffledIndicies, indicies, slot)
 
 	return shuffledIndicies[start:end], nil
 }
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 65613ccb419..61910d88787 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -48,9 +48,6 @@ type HistoricalStatesReader struct {
 	blockReader    freezeblocks.BeaconSnapshotReader
 	stateSn        *freezeblocks.CaplinStateSnapshots
 	genesisState   *state.CachingBeaconState
-
-	// cache for shuffled sets
-	shuffledSetsCache *lru.Cache[uint64, []uint64]
 }
 
 func NewHistoricalStatesReader(

From 69b18379b80fcafb31f2636cc0352b3c519bf180 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 18:48:36 +0100
Subject: [PATCH 113/131] save

---
 .../historical_states_reader.go                 | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 61910d88787..1b50b23ccb3 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -33,7 +33,6 @@ import (
 	"github.com/erigontech/erigon/cl/persistence/base_encoding"
 	state_accessors "github.com/erigontech/erigon/cl/persistence/state"
 	"github.com/erigontech/erigon/cl/phase1/core/state"
-	"github.com/erigontech/erigon/cl/phase1/core/state/lru"
 	"github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 	"github.com/klauspost/compress/zstd"
 )
@@ -56,18 +55,12 @@ func NewHistoricalStatesReader(
 	validatorTable *state_accessors.StaticValidatorTable,
 	genesisState *state.CachingBeaconState, stateSn *freezeblocks.CaplinStateSnapshots) *HistoricalStatesReader {
 
-	cache, err := lru.New[uint64, []uint64]("shuffledSetsCache_reader", 125)
-	if err != nil {
-		panic(err)
-	}
-
 	return &HistoricalStatesReader{
-		cfg:               cfg,
-		blockReader:       blockReader,
-		genesisState:      genesisState,
-		validatorTable:    validatorTable,
-		shuffledSetsCache: cache,
-		stateSn:           stateSn,
+		cfg:            cfg,
+		blockReader:    blockReader,
+		genesisState:   genesisState,
+		validatorTable: validatorTable,
+		stateSn:        stateSn,
 	}
 }
 

From 4397e10e3d33808757dca8d79c4f0da237880537 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 22:57:57 +0100
Subject: [PATCH 114/131] save

---
 cl/antiquary/state_antiquary_test.go          |  2 +-
 cl/beacon/handler/states_test.go              | 29 ++++++-
 .../historical_states_reader.go               |  3 +-
 holy.txt                                      | 32 -------
 .../freezeblocks/caplin_state_snapshots.go    | 87 +++++++++++--------
 5 files changed, 83 insertions(+), 70 deletions(-)
 delete mode 100644 holy.txt

diff --git a/cl/antiquary/state_antiquary_test.go b/cl/antiquary/state_antiquary_test.go
index 08e37c4fc6c..12f8cf8d792 100644
--- a/cl/antiquary/state_antiquary_test.go
+++ b/cl/antiquary/state_antiquary_test.go
@@ -41,7 +41,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, false, nil)
+	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 }
 
diff --git a/cl/beacon/handler/states_test.go b/cl/beacon/handler/states_test.go
index c822ed6774f..48035a6e599 100644
--- a/cl/beacon/handler/states_test.go
+++ b/cl/beacon/handler/states_test.go
@@ -18,6 +18,7 @@ package handler
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -159,6 +160,7 @@ func TestGetStateFullHistorical(t *testing.T) {
 	// setupTestingHandler(t, clparams.Phase0Version)
 	_, blocks, _, _, postState, handler, _, _, fcu, _ := setupTestingHandler(t, clparams.Phase0Version, log.Root(), true)
 
+	fmt.Println("AX")
 	postRoot, err := postState.HashSSZ()
 	require.NoError(t, err)
 
@@ -213,7 +215,32 @@ func TestGetStateFullHistorical(t *testing.T) {
 			require.NoError(t, err)
 			other := state.New(&clparams.MainnetBeaconConfig)
 			require.NoError(t, other.DecodeSSZ(out, int(clparams.Phase0Version)))
-
+			for i := 0; i < other.ValidatorLength(); i++ {
+				if other.ValidatorSet().Get(i).PublicKey() != postState.ValidatorSet().Get(i).PublicKey() {
+					fmt.Println("difference in validator", i, other.ValidatorSet().Get(i).PublicKey(), postState.ValidatorSet().Get(i).PublicKey())
+				}
+				if other.ValidatorSet().Get(i).WithdrawalCredentials() != postState.ValidatorSet().Get(i).WithdrawalCredentials() {
+					fmt.Println("difference in withdrawal", i, other.ValidatorSet().Get(i).WithdrawalCredentials(), postState.ValidatorSet().Get(i).WithdrawalCredentials())
+				}
+				if other.ValidatorSet().Get(i).EffectiveBalance() != postState.ValidatorSet().Get(i).EffectiveBalance() {
+					fmt.Println("difference in effective", i, other.ValidatorSet().Get(i).EffectiveBalance(), postState.ValidatorSet().Get(i).EffectiveBalance())
+				}
+				if other.ValidatorSet().Get(i).Slashed() != postState.ValidatorSet().Get(i).Slashed() {
+					fmt.Println("difference in slashed", i, other.ValidatorSet().Get(i).Slashed(), postState.ValidatorSet().Get(i).Slashed())
+				}
+				if other.ValidatorSet().Get(i).ActivationEligibilityEpoch() != postState.ValidatorSet().Get(i).ActivationEligibilityEpoch() {
+					fmt.Println("difference in activation", i, other.ValidatorSet().Get(i).ActivationEligibilityEpoch(), postState.ValidatorSet().Get(i).ActivationEligibilityEpoch())
+				}
+				if other.ValidatorSet().Get(i).ActivationEpoch() != postState.ValidatorSet().Get(i).ActivationEpoch() {
+					fmt.Println("difference in activation", i, other.ValidatorSet().Get(i).ActivationEpoch(), postState.ValidatorSet().Get(i).ActivationEpoch())
+				}
+				if other.ValidatorSet().Get(i).ExitEpoch() != postState.ValidatorSet().Get(i).ExitEpoch() {
+					fmt.Println("difference in exit", i, other.ValidatorSet().Get(i).ExitEpoch(), postState.ValidatorSet().Get(i).ExitEpoch())
+				}
+				if other.ValidatorSet().Get(i).WithdrawableEpoch() != postState.ValidatorSet().Get(i).WithdrawableEpoch() {
+					fmt.Println("difference in withdrawable", i, other.ValidatorSet().Get(i).WithdrawableEpoch(), postState.ValidatorSet().Get(i).WithdrawableEpoch())
+				}
+			}
 			otherRoot, err := other.HashSSZ()
 			require.NoError(t, err)
 			require.Equal(t, postRoot, otherRoot)
diff --git a/cl/persistence/state/historical_states_reader/historical_states_reader.go b/cl/persistence/state/historical_states_reader/historical_states_reader.go
index 894370a7235..9f4a04d1f54 100644
--- a/cl/persistence/state/historical_states_reader/historical_states_reader.go
+++ b/cl/persistence/state/historical_states_reader/historical_states_reader.go
@@ -451,6 +451,7 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, kvGetter
 		return nil, err
 	}
 	forward := remainder <= midpoint || currentStageProgress <= freshDumpSlot+clparams.SlotsPerDump
+	fmt.Println("forward", forward)
 	if forward {
 		compressed, err = kvGetter(dumpBucket, base_encoding.Encode64ToBytes4(freshDumpSlot))
 		if err != nil {
@@ -491,7 +492,7 @@ func (r *HistoricalStatesReader) reconstructDiffedUint64List(tx kv.Tx, kvGetter
 		return nil, err
 	}
 	if forward {
-		for currSlot := freshDumpSlot; currSlot <= slot && currSlot < highestSlotAvailable; currSlot++ {
+		for currSlot := freshDumpSlot; currSlot <= slot && currSlot <= highestSlotAvailable; currSlot++ {
 			key := base_encoding.Encode64ToBytes4(currSlot)
 			v, err := kvGetter(diffBucket, key)
 			if err != nil {
diff --git a/holy.txt b/holy.txt
deleted file mode 100644
index 921062010cf..00000000000
--- a/holy.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-0 0x607db06200000000000000000000000000000000000000000000000000000000                                                                                                                                                  
-1 0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078                                                                                                                                                  
-2 0x41420f0000000000000000000000000000000000000000000000000000000000                                                                                                                                                  
-3 0x30680b18ada5ceaeb222d0d15bd7058fbc4e1e5e3c002d504adee82cc013a517                                                                                                                                                  
-4 0x1158a7769cb0dfc3d015aa03bd76760d57bab8ad2fb77885a07917f1754194cf                                                                                                                                                  
-5 0xfe1acfde1678da01248a9825ec08d4f3ee73615c1263095077ee23063d1819cb                                                                                                                                                  
-6 0x0ead07fe6e6026ee73f966aceffaaeff6b50d6df923e5230a9cc33023d52adbe                                                                                                                                                  
-7 0xb6049c0ff9220d02083c30474400cbcb358a6d6804082757553468badccd8a48                                                                                                                                                  
-8 0xa4007911445daf00197d2cca907cebf0529773561b702a2136decb52edc87348                                                                                                                                                  
-9 0xc231c219f35945a38dbab773bf4f32c9080c59f9e6284fbb92ffe02c3318165d
-10 0x9301000000000000000000000000000000000000000000000000000000000000
-11 0xed774cba995d2b5cd55a05ac5126d4fb24a1dbf7523ebdc0b1dc8f7f02de30d6
-12 0xe7c27b3e6aaccb0c4f42eda4ffadd5f92fc28edfa5ef573f28ab5495f669282a
-13 0xe65a9db1e18c1435832c978ca666c5502dab9b85bf1258623e8a2e96e986e78f
-14 0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220
-15 0xe15e05e8cb173b5d2a4bc5acea5aab50ea2e616fcc41bee9833377ce40135897
-16 0xe9eddec5525f068001fdff036039e7946248a1d80ce8f6b40a1e79d01303519b
-17 0x0f00000000000000000000000000000000000000000000000000000000000000
-18 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
-19 0xe74aa97c38fea5a58af5514cd78e2180f0ba62f0709676c62d6e9af62fff77d0
-20 0xa9ff63a992f9e2d2894c8947050fe7360e3811fb9e1468ffde2ca6e4413aa089
-21 0x3d4be5d019ba15ea3ef304a83b8a067f2e79f46a3fac8069306a6c814a0a35eb
-22 0x2961343e46fbbea5ef5373a5db989b5e2ed19baef61d9f26ee6ac63a9023862b
-23 0x3843792806b2d08dd125002fd3e68d117c7860e4c863f7f60d59625a6f220d48
-24 0x69c41e47f10b14f920a3851f8a4f71d19f83a8d1e28204efccccd644747141be
-25 0x0000000000000000000000000000000000000000000000000000000000000000
-26 0x0000000000000000000000000000000000000000000000000000000000000000
-27 0x0000000000000000000000000000000000000000000000000000000000000000
-28 0x0000000000000000000000000000000000000000000000000000000000000000
-29 0x0000000000000000000000000000000000000000000000000000000000000000
-30 0x0000000000000000000000000000000000000000000000000000000000000000
-31 0x0000000000000000000000000000000000000000000000000000000000000000
diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 10e286d91b7..0ff7714c836 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -103,7 +103,13 @@ type CaplinStateSnapshots struct {
 
 	// BeaconBlocks *segments
 	// BlobSidecars *segments
-	Segments      map[string]*segments
+	// Segments      map[string]*segments
+	dirtyLock sync.RWMutex                            // guards `dirty` field
+	dirty     map[string]*btree.BTreeG[*DirtySegment] // ordered map `type.Enum()` -> DirtySegments
+
+	visibleLock sync.RWMutex               // guards  `visible` field
+	visible     map[string]VisibleSegments // ordered map `type.Enum()` -> VisbileSegments
+
 	snapshotTypes SnapshotTypes
 
 	dir         string
@@ -137,13 +143,21 @@ func NewCaplinStateSnapshots(cfg ethconfig.BlocksFreezing, beaconCfg *clparams.B
 	// BlobSidecars := &segments{
 	// 	DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
 	// }
-	Segments := make(map[string]*segments)
+	// Segments := make(map[string]*segments)
+	// for k := range snapshotTypes.KeyValueGetters {
+	// 	Segments[k] = &segments{
+	// 		DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
+	// 	}
+	// }
+	dirty := make(map[string]*btree.BTreeG[*DirtySegment])
 	for k := range snapshotTypes.KeyValueGetters {
-		Segments[k] = &segments{
-			DirtySegments: btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false}),
-		}
+		dirty[k] = btree.NewBTreeGOptions[*DirtySegment](DirtySegmentLess, btree.Options{Degree: 128, NoLocks: false})
+	}
+	visible := make(map[string]VisibleSegments)
+	for k := range snapshotTypes.KeyValueGetters {
+		visible[k] = make(VisibleSegments, 0)
 	}
-	c := &CaplinStateSnapshots{snapshotTypes: snapshotTypes, dir: dirs.SnapCaplin, tmpdir: dirs.Tmp, cfg: cfg, Segments: Segments, logger: logger, beaconCfg: beaconCfg}
+	c := &CaplinStateSnapshots{snapshotTypes: snapshotTypes, dir: dirs.SnapCaplin, tmpdir: dirs.Tmp, cfg: cfg, visible: visible, dirty: dirty, logger: logger, beaconCfg: beaconCfg}
 	c.recalcVisibleFiles()
 	return c
 }
@@ -165,8 +179,8 @@ func (s *CaplinStateSnapshots) LS() {
 
 	for _, roTx := range view.roTxs {
 		if roTx != nil {
-			for _, seg := range roTx.VisibleSegments {
-				s.logger.Info("[agg] ", "f", seg.src.Decompressor.FileName(), "words", seg.src.Decompressor.Count())
+			for _, seg := range roTx.segments {
+				s.logger.Info("[agg] ", "f", seg.src.filePath, "words", seg.src.Decompressor.Count())
 			}
 		}
 	}
@@ -182,11 +196,13 @@ func (s *CaplinStateSnapshots) SegFileNames(from, to uint64) []string {
 		if roTx == nil {
 			continue
 		}
-		for _, seg := range roTx.VisibleSegments {
-			if seg.from >= from && seg.to <= to {
-				res = append(res, seg.src.FileName())
+		for _, seg := range roTx.segments {
+			if seg.from >= to || seg.to <= from {
+				continue
 			}
+			res = append(res, seg.src.filePath)
 		}
+
 	}
 	return res
 }
@@ -235,12 +251,12 @@ Loop:
 		var exists bool
 		var sn *DirtySegment
 
-		segments, ok := s.Segments[f.TypeString]
+		dirtySegments, ok := s.dirty[f.TypeString]
 		if !ok {
 			continue
 		}
 		filePath := filepath.Join(s.dir, fName)
-		segments.DirtySegments.Walk(func(segments []*DirtySegment) bool {
+		dirtySegments.Walk(func(segments []*DirtySegment) bool {
 			for _, sn2 := range segments {
 				if sn2.Decompressor == nil { // it's ok if some segment was not able to open
 					continue
@@ -281,7 +297,7 @@ Loop:
 		if !exists {
 			// it's possible to iterate over .seg file even if you don't have index
 			// then make segment available even if index open may fail
-			segments.DirtySegments.Set(sn)
+			dirtySegments.Set(sn)
 		}
 		if err := openIdxForCaplinStateIfNeeded(sn, filePath, optimistic); err != nil {
 			return err
@@ -358,8 +374,8 @@ func (s *CaplinStateSnapshots) recalcVisibleFiles() {
 		s.indicesReady.Store(true)
 	}()
 
-	s.visibleSegmentsLock.Lock()
-	defer s.visibleSegmentsLock.Unlock()
+	s.visibleLock.Lock()
+	defer s.visibleLock.Unlock()
 
 	getNewVisibleSegments := func(dirtySegments *btree.BTreeG[*DirtySegment]) []*VisibleSegment {
 		newVisibleSegments := make([]*VisibleSegment, 0, dirtySegments.Len())
@@ -386,27 +402,28 @@ func (s *CaplinStateSnapshots) recalcVisibleFiles() {
 		return newVisibleSegments
 	}
 
-	for _, segments := range s.Segments {
-		segments.VisibleSegments = getNewVisibleSegments(segments.DirtySegments)
-		var maxIdx uint64
-		if len(segments.VisibleSegments) > 0 {
-			maxIdx = segments.VisibleSegments[len(segments.VisibleSegments)-1].to - 1
-		}
-		segments.maxVisibleBlock.Store(maxIdx)
+	for k := range s.visible {
+		s.visible[k] = getNewVisibleSegments(s.dirty[k])
 	}
 }
 
 func (s *CaplinStateSnapshots) idxAvailability() uint64 {
-	minVisible := uint64(math.MaxUint64)
-	for _, segments := range s.Segments {
-		if segments.maxVisibleBlock.Load() < minVisible {
-			minVisible = segments.maxVisibleBlock.Load()
+	s.visibleLock.RLock()
+	defer s.visibleLock.RUnlock()
+
+	min := uint64(math.MaxUint64)
+	for _, segs := range s.visible {
+		if len(segs) == 0 {
+			return 0
+		}
+		if segs[len(segs)-1].to < min {
+			min = segs[len(segs)-1].to
 		}
 	}
-	if minVisible == math.MaxUint64 {
+	if min == math.MaxUint64 {
 		return 0
 	}
-	return minVisible
+	return min
 }
 
 func listAllSegFilesInDir(dir string) []string {
@@ -438,9 +455,9 @@ func (s *CaplinStateSnapshots) closeWhatNotInList(l []string) {
 		protectFiles[fName] = struct{}{}
 	}
 
-	for _, segments := range s.Segments {
+	for _, dirtySegments := range s.dirty {
 		toClose := make([]*DirtySegment, 0)
-		segments.DirtySegments.Walk(func(segments []*DirtySegment) bool {
+		dirtySegments.Walk(func(segments []*DirtySegment) bool {
 			for _, sn := range segments {
 				if sn.Decompressor == nil {
 					continue
@@ -455,7 +472,7 @@ func (s *CaplinStateSnapshots) closeWhatNotInList(l []string) {
 		})
 		for _, sn := range toClose {
 			sn.close()
-			segments.DirtySegments.Delete(sn)
+			dirtySegments.Delete(sn)
 		}
 	}
 }
@@ -478,7 +495,7 @@ func (s *CaplinStateSnapshots) View() *CaplinStateView {
 	s.dirtySegmentsLock.RLock()
 	defer s.dirtySegmentsLock.RUnlock()
 
-	for k, segments := range s.Segments {
+	for k, segments := range s.visible {
 		v.roTxs[k] = segments.BeginRotx()
 	}
 	return v
@@ -499,10 +516,10 @@ func (v *CaplinStateView) Close() {
 }
 
 func (v *CaplinStateView) VisibleSegments(tbl string) []*VisibleSegment {
-	if v.s == nil || v.s.Segments[tbl] == nil {
+	if v.s == nil || v.s.visible[tbl] == nil {
 		return nil
 	}
-	return v.s.Segments[tbl].VisibleSegments
+	return v.s.visible[tbl]
 }
 
 func (v *CaplinStateView) VisibleSegment(slot uint64, tbl string) (*VisibleSegment, bool) {

From 536ed7bfcbef2c8c2c783cab69675234a8e1ede6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:08:08 +0100
Subject: [PATCH 115/131] save

---
 cl/antiquary/state_antiquary.go     | 19 ++++++++++++++++---
 erigon-lib/downloader/downloader.go |  6 +++++-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/cl/antiquary/state_antiquary.go b/cl/antiquary/state_antiquary.go
index 24d04604f81..667ba909cc1 100644
--- a/cl/antiquary/state_antiquary.go
+++ b/cl/antiquary/state_antiquary.go
@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"runtime"
 	"sync"
 	"time"
 
@@ -30,6 +29,7 @@ import (
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/downloader/snaptype"
 	"github.com/erigontech/erigon-lib/etl"
+	proto_downloader "github.com/erigontech/erigon-lib/gointerfaces/downloaderproto"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/cl/clparams"
@@ -510,12 +510,25 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 			blocksPerStatefulFile,
 			s.sn.Salt,
 			s.dirs,
-			runtime.NumCPU(),
+			1,
 			log.LvlInfo,
 			s.logger,
 		); err != nil {
 			return err
 		}
+		paths := s.stateSn.SegFileNames(from, to)
+		downloadItems := make([]*proto_downloader.AddItem, len(paths))
+		for i, path := range paths {
+			downloadItems[i] = &proto_downloader.AddItem{
+				Path: path,
+			}
+		}
+		if s.downloader != nil {
+			// Notify bittorent to seed the new snapshots
+			if _, err := s.downloader.Add(s.ctx, &proto_downloader.AddRequest{Items: downloadItems}); err != nil {
+				s.logger.Warn("[Antiquary] Failed to add items to bittorent", "err", err)
+			}
+		}
 		if err := s.stateSn.OpenFolder(); err != nil {
 			return err
 		}
@@ -576,7 +589,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 		if err != nil {
 			return fmt.Errorf("failed to read historical state at slot %d: %w", attempt, err)
 		}
-		fmt.Println("currentState", s.currentState, attempt)
+
 		if s.currentState == nil {
 			log.Warn("historical state not found, backoff more and try again", "slot", attempt)
 			backoffStep += backoffStrides
diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index cf89315feb3..a2f5693bbfd 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2652,7 +2652,11 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 			return nil, err
 		}
 	}
-	files = append(append(append(append(files, l1...), l2...), l3...), l4...)
+	l5, err := seedableStateFilesBySubDir(dirs.Snap, "caplin", all)
+	if err != nil {
+		return nil, err
+	}
+	files = append(append(append(append(append(files, l1...), l2...), l3...), l4...), l5...)
 	return files, nil
 }
 

From 7ba71b2270ea88faec97f47b69fc82cdb50553ae Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:11:59 +0100
Subject: [PATCH 116/131] save

---
 erigon-lib/downloader/downloader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index a2f5693bbfd..883019e806d 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2652,7 +2652,7 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 			return nil, err
 		}
 	}
-	l5, err := seedableStateFilesBySubDir(dirs.Snap, "caplin", all)
+	l5, err := seedableSegmentFiles(dirs.Snap, "caplin", all)
 	if err != nil {
 		return nil, err
 	}

From 2c562b14c3b7eff0deaec97e380a36381efa0499 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:13:58 +0100
Subject: [PATCH 117/131] save

---
 erigon-lib/downloader/downloader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index 883019e806d..dd6a5a7aa09 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2652,7 +2652,7 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 			return nil, err
 		}
 	}
-	l5, err := seedableSegmentFiles(dirs.Snap, "caplin", all)
+	l5, err := seedableSegmentFiles(path.Join(dirs.Snap, "caplin"), chainName, all)
 	if err != nil {
 		return nil, err
 	}

From f1cd894b3db9dad29df821520038c91be2528e70 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:16:12 +0100
Subject: [PATCH 118/131] save

---
 erigon-lib/downloader/downloader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index dd6a5a7aa09..7481aa8e4b5 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2652,7 +2652,7 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 			return nil, err
 		}
 	}
-	l5, err := seedableSegmentFiles(path.Join(dirs.Snap, "caplin"), chainName, all)
+	l5, err := seedableSegmentFiles(dirs.SnapCaplin, chainName, all)
 	if err != nil {
 		return nil, err
 	}

From 1066312c99ebc67da8fdc7448cdd9b4f42df6cbd Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:17:31 +0100
Subject: [PATCH 119/131] save

---
 erigon-lib/downloader/downloader.go | 1 +
 erigon-lib/downloader/util.go       | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index 7481aa8e4b5..ca2f31aaa88 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2652,6 +2652,7 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 			return nil, err
 		}
 	}
+	fmt.Println(dirs.SnapCaplin)
 	l5, err := seedableSegmentFiles(dirs.SnapCaplin, chainName, all)
 	if err != nil {
 		return nil, err
diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index 5dc141ddbca..e05e2afbf97 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -79,7 +79,7 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 
 	res := make([]string, 0, len(files))
 	for _, fPath := range files {
-
+		fmt.Println(fPath)
 		_, name := filepath.Split(fPath)
 		if !skipSeedableCheck && !snaptype.IsCorrectFileName(name) {
 			continue

From b903cf2e390582996f1f3eda6f745b7ccc5dbef4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:19:13 +0100
Subject: [PATCH 120/131] save

---
 erigon-lib/downloader/util.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index e05e2afbf97..19809533877 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -81,6 +81,10 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 	for _, fPath := range files {
 		fmt.Println(fPath)
 		_, name := filepath.Split(fPath)
+		if strings.Contains(dir, "caplin") {
+			res = append(res, name)
+			continue
+		}
 		if !skipSeedableCheck && !snaptype.IsCorrectFileName(name) {
 			continue
 		}

From 84c6bc356c781697779e2bd83e0be7abdab50994 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:19:26 +0100
Subject: [PATCH 121/131] save

---
 erigon-lib/downloader/util.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index 19809533877..129020abf75 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -81,6 +81,7 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 	for _, fPath := range files {
 		fmt.Println(fPath)
 		_, name := filepath.Split(fPath)
+		// A bit hacky but whatever...
 		if strings.Contains(dir, "caplin") {
 			res = append(res, name)
 			continue

From 97b50eaedbf937c1379358177573a91949af667c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:19:41 +0100
Subject: [PATCH 122/131] save

---
 erigon-lib/downloader/util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index 129020abf75..99f25f20cba 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -81,7 +81,7 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 	for _, fPath := range files {
 		fmt.Println(fPath)
 		_, name := filepath.Split(fPath)
-		// A bit hacky but whatever...
+		// A bit hacky but whatever... basically caplin is incompatible with enums.
 		if strings.Contains(dir, "caplin") {
 			res = append(res, name)
 			continue

From b0609b1dbb28408e251ffe26f51256198921ff31 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:21:23 +0100
Subject: [PATCH 123/131] save

---
 erigon-lib/downloader/downloader.go | 1 +
 erigon-lib/downloader/util.go       | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index ca2f31aaa88..019185d93f1 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2657,6 +2657,7 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(l5)
 	files = append(append(append(append(append(files, l1...), l2...), l3...), l4...), l5...)
 	return files, nil
 }
diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index 99f25f20cba..a3747684a70 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -79,7 +79,6 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 
 	res := make([]string, 0, len(files))
 	for _, fPath := range files {
-		fmt.Println(fPath)
 		_, name := filepath.Split(fPath)
 		// A bit hacky but whatever... basically caplin is incompatible with enums.
 		if strings.Contains(dir, "caplin") {

From 082707243ee7780dd51c7fe29383d9f995702877 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:23:41 +0100
Subject: [PATCH 124/131] save

---
 erigon-lib/downloader/util.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
index a3747684a70..3887820a23a 100644
--- a/erigon-lib/downloader/util.go
+++ b/erigon-lib/downloader/util.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -82,7 +83,7 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 		_, name := filepath.Split(fPath)
 		// A bit hacky but whatever... basically caplin is incompatible with enums.
 		if strings.Contains(dir, "caplin") {
-			res = append(res, name)
+			res = append(res, path.Join("caplin", name))
 			continue
 		}
 		if !skipSeedableCheck && !snaptype.IsCorrectFileName(name) {

From 18f5a384c2e6d22fa254415dd08e167f4b70da2b Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:24:45 +0100
Subject: [PATCH 125/131] save

---
 erigon-lib/downloader/downloader.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index 019185d93f1..ca2f31aaa88 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2657,7 +2657,6 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 	if err != nil {
 		return nil, err
 	}
-	fmt.Println(l5)
 	files = append(append(append(append(append(files, l1...), l2...), l3...), l4...), l5...)
 	return files, nil
 }

From 00dffea501a84e0eb90a192b9836fd3826f14aff Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:32:00 +0100
Subject: [PATCH 126/131] save

---
 turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
index 0ff7714c836..da4942ba61f 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_state_snapshots.go
@@ -542,7 +542,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 
 	compressCfg := seg.DefaultCfg
 	compressCfg.Workers = workers
-	sn, err := seg.NewCompressor(ctx, fmt.Sprintf("Snapshots %s", snapName), f.Path, tmpDir, compressCfg, lvl, logger)
+	sn, err := seg.NewCompressor(ctx, "Snapshots "+snapName, f.Path, tmpDir, compressCfg, lvl, logger)
 	if err != nil {
 		return err
 	}
@@ -556,7 +556,7 @@ func dumpCaplinState(ctx context.Context, snapName string, kvGetter KeyValueGett
 			return err
 		}
 		if i%20_000 == 0 {
-			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "progress", i)
+			logger.Log(lvl, "Dumping "+snapName, "progress", i)
 		}
 		if compress {
 			if err := sn.AddWord(dump); err != nil {
@@ -620,7 +620,7 @@ func (s *CaplinStateSnapshots) DumpCaplinState(ctx context.Context, fromSlot, to
 			}
 			// keep beaconblocks here but whatever....
 			to := i + blocksPerFile
-			logger.Log(lvl, fmt.Sprintf("Dumping %s", snapName), "from", i, "to", to)
+			logger.Log(lvl, "Dumping "+snapName, "from", i, "to", to)
 			if err := dumpCaplinState(ctx, snapName, kvGetter, i, to, blocksPerFile, salt, dirs, workers, lvl, logger, s.snapshotTypes.Compression[snapName]); err != nil {
 				return err
 			}

From 006e5b717c4c235ee3f3836ecf36b688e4a5180a Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:44:03 +0100
Subject: [PATCH 127/131] save

---
 erigon-lib/downloader/downloader.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
index ca2f31aaa88..4b037c45ffd 100644
--- a/erigon-lib/downloader/downloader.go
+++ b/erigon-lib/downloader/downloader.go
@@ -2645,17 +2645,19 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 	if err != nil {
 		return nil, err
 	}
-	var l4 []string
+	var l4, l5 []string
 	if all {
 		l4, err = seedableStateFilesBySubDir(dirs.Snap, "accessor", all)
 		if err != nil {
 			return nil, err
 		}
 	}
-	fmt.Println(dirs.SnapCaplin)
-	l5, err := seedableSegmentFiles(dirs.SnapCaplin, chainName, all)
-	if err != nil {
-		return nil, err
+	// check if dirs.SnapCaplin exists
+	if _, err := os.Stat(dirs.SnapCaplin); !os.IsNotExist(err) {
+		l5, err = seedableSegmentFiles(dirs.SnapCaplin, chainName, all)
+		if err != nil {
+			return nil, err
+		}
 	}
 	files = append(append(append(append(append(files, l1...), l2...), l3...), l4...), l5...)
 	return files, nil

From 7405711047492761850bf383abd69c1b87f7dcc9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Sun, 27 Oct 2024 23:44:25 +0100
Subject: [PATCH 128/131] save

---
 erigon-lib/common/datadir/dirs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/common/datadir/dirs.go b/erigon-lib/common/datadir/dirs.go
index 3a654eead38..9d35da1550c 100644
--- a/erigon-lib/common/datadir/dirs.go
+++ b/erigon-lib/common/datadir/dirs.go
@@ -73,6 +73,7 @@ func New(datadir string) Dirs {
 		SnapHistory:     filepath.Join(datadir, "snapshots", "history"),
 		SnapDomain:      filepath.Join(datadir, "snapshots", "domain"),
 		SnapAccessors:   filepath.Join(datadir, "snapshots", "accessor"),
+		SnapCaplin:      filepath.Join(datadir, "snapshots", "caplin"),
 		Downloader:      filepath.Join(datadir, "downloader"),
 		TxPool:          filepath.Join(datadir, "txpool"),
 		Nodes:           filepath.Join(datadir, "nodes"),
@@ -80,7 +81,6 @@ func New(datadir string) Dirs {
 		CaplinIndexing:  filepath.Join(datadir, "caplin", "indexing"),
 		CaplinLatest:    filepath.Join(datadir, "caplin", "latest"),
 		CaplinGenesis:   filepath.Join(datadir, "caplin", "genesis"),
-		SnapCaplin:      filepath.Join(datadir, "snapshots", "caplin"),
 	}
 
 	dir.MustExist(dirs.Chaindata, dirs.Tmp,

From 9930833cd99f5b1a26a8925575e9b67cf10897be Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 28 Oct 2024 00:23:12 +0100
Subject: [PATCH 129/131] save

---
 erigon-lib/common/datadir/dirs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erigon-lib/common/datadir/dirs.go b/erigon-lib/common/datadir/dirs.go
index 9d35da1550c..ca6b975552c 100644
--- a/erigon-lib/common/datadir/dirs.go
+++ b/erigon-lib/common/datadir/dirs.go
@@ -84,7 +84,7 @@ func New(datadir string) Dirs {
 	}
 
 	dir.MustExist(dirs.Chaindata, dirs.Tmp,
-		dirs.SnapIdx, dirs.SnapHistory, dirs.SnapDomain, dirs.SnapAccessors,
+		dirs.SnapIdx, dirs.SnapHistory, dirs.SnapDomain, dirs.SnapAccessors, dirs.SnapCaplin,
 		dirs.Downloader, dirs.TxPool, dirs.Nodes, dirs.CaplinBlobs, dirs.CaplinIndexing, dirs.CaplinLatest, dirs.CaplinGenesis)
 	return dirs
 }

From 9bcb2fff0d50ed1fbfd4f30d3ea88df60e82bcad Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Wed, 30 Oct 2024 00:43:17 +0100
Subject: [PATCH 130/131] save

---
 turbo/snapshotsync/caplin_state_snapshots.go | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/turbo/snapshotsync/caplin_state_snapshots.go b/turbo/snapshotsync/caplin_state_snapshots.go
index cd18b75fae2..0404796e668 100644
--- a/turbo/snapshotsync/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/caplin_state_snapshots.go
@@ -24,6 +24,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -690,3 +691,21 @@ func (s *CaplinStateSnapshots) BuildMissingIndices(ctx context.Context, logger l
 
 	return s.OpenFolder()
 }
+
+func (s *CaplinStateSnapshots) Get(tbl string, slot uint64) ([]byte, error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			panic(fmt.Sprintf("Get(%s, %d), %s, %s\n", tbl, slot, rec, debug.Stack()))
+		}
+	}()
+
+	view := s.View()
+	defer view.Close()
+
+	seg, ok := view.VisibleSegment(slot, tbl)
+	if !ok {
+		return nil, nil
+	}
+
+	return seg.Get(slot)
+}

From fad80fe542870be4df5e67cee8bf137a1dff54fe Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Wed, 30 Oct 2024 00:44:27 +0100
Subject: [PATCH 131/131] save

---
 turbo/snapshotsync/caplin_state_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/caplin_state_snapshots.go b/turbo/snapshotsync/caplin_state_snapshots.go
index 0404796e668..f5c75568dcc 100644
--- a/turbo/snapshotsync/caplin_state_snapshots.go
+++ b/turbo/snapshotsync/caplin_state_snapshots.go
@@ -82,6 +82,7 @@ func getKvGetterForStateTable(db kv.RoDB, tableName string) KeyValueGetter {
 		if err := db.View(context.TODO(), func(tx kv.Tx) error {
 			key = base_encoding.Encode64ToBytes4(numId)
 			value, err = tx.GetOne(tableName, base_encoding.Encode64ToBytes4(numId))
+			value = libcommon.Copy(value)
 			return err
 		}); err != nil {
 			return nil, nil, err