Merge branch 'op-erigon' into increase-fcu-timeout

README.md

@@ -89,9 +89,9 @@ If your chain is the one of `op-mainnet` & `op-sepolia`, or your chain is synced
 **[Optional]** 
 Disables transaction pool gossiping. Though this is not required, it's useful to set this to true since transaction pool gossip is currently unsupported in the Optimism protocol. If not provided, default value is set to `false`.
 
-### `--maxpeers=0`, `--nodiscover`
+### `--maxpeers=0`, `--nodiscover`, `--v5disc=false`
 **[Optional]** 
-Disable P2P. Execution-layer peering is currently not supported in the Optimism protocol. Though this is not required, it saves resources since TX pool gossip is currently not available.
+Disable P2P. This can save resources if you are only using op-node to sync the chain instead of using execution-layer syncing.  
 
 ## Support Chains
 op-erigon supports every OP Stack chains listed in [superchain-registry](https://github.com/ethereum-optimism/superchain-registry).
@@ -134,8 +134,7 @@ $ ./build/bin/erigon \
     --rollup.historicalrpc="https://mainnet.optimism.io" \
     --txpool.gossip.disable=true \
     --chain=op-mainnet \
-    --db.size.limit=8TB \
-    --nodiscover
+    --db.size.limit=8TB
 ```
 2. Use the Docker image: You can get the official Docker image from [testinprod/op-erigon](https://hub.docker.com/r/testinprod/op-erigon).
 3. Use the Helm chart: If you want to deploy op-erigon to the K8S cluster, you can use [Helm chart](https://artifacthub.io/packages/helm/op-charts/erigon).

cl/beacon/beaconhttp/args.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/ledgerwatch/erigon-lib/common"
@@ -168,9 +169,13 @@ func Uint64FromQueryParams(r *http.Request, name string) (*uint64, error) {
 
 // decode a list of strings from the query params
 func StringListFromQueryParams(r *http.Request, name string) ([]string, error) {
-	str := r.URL.Query().Get(name)
-	if str == "" {
+	values := r.URL.Query()[name]
+	if len(values) == 0 {
 		return nil, nil
 	}
+
+	// Combine all values into a single string, separating by comma
+	str := strings.Join(values, ",")
+
 	return regexp.MustCompile(`\s*,\s*`).Split(str, -1), nil
 }

cl/beacon/handler/blobs.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/ledgerwatch/erigon/cl/beacon/beaconhttp"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -51,13 +52,33 @@ func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.R
 	if err != nil {
 		return nil, err
 	}
-
+	strIdxs, err := beaconhttp.StringListFromQueryParams(r, "indices")
+	if err != nil {
+		return nil, err
+	}
 	resp := solid.NewStaticListSSZ[*cltypes.BlobSidecar](696969, blobSidecarSSZLenght)
 	if !found {
 		return beaconhttp.NewBeaconResponse(resp), nil
 	}
-	for _, v := range out {
-		resp.Append(v)
+	if len(strIdxs) == 0 {
+		for _, v := range out {
+			resp.Append(v)
+		}
+	} else {
+		included := make(map[uint64]struct{})
+		for _, idx := range strIdxs {
+			i, err := strconv.ParseUint(idx, 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			included[i] = struct{}{}
+		}
+		for _, v := range out {
+			if _, ok := included[v.Index]; ok {
+				resp.Append(v)
+			}
+		}
 	}
+
 	return beaconhttp.NewBeaconResponse(resp), nil
 }

cl/clparams/config.go

@@ -15,6 +15,7 @@ package clparams
 
 import (
 	"crypto/rand"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -315,7 +316,9 @@ func (b ConfigByte) MarshalJSON() ([]byte, error) {
 type ConfigForkVersion uint32
 
 func (v ConfigForkVersion) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"0x%x\"", v)), nil
+	tmp := make([]byte, 4)
+	binary.BigEndian.PutUint32(tmp, uint32(v))
+	return []byte(fmt.Sprintf("\"0x%x\"", tmp)), nil
 }
 
 // BeaconChainConfig contains constant configs for node to participate in beacon chain.

cl/sentinel/service/service.go

@@ -96,7 +96,8 @@ func (s *SentinelServer) PublishGossip(_ context.Context, msg *sentinelrpc.Gossi
 		gossip.TopicNameVoluntaryExit,
 		gossip.TopicNameProposerSlashing,
 		gossip.TopicNameSyncCommitteeContributionAndProof,
-		gossip.TopicNameAttesterSlashing:
+		gossip.TopicNameAttesterSlashing,
+		gossip.TopicNameBlsToExecutionChange:
 		subscription = manager.GetMatchingSubscription(msg.Name)
 	default:
 		// check subnets

cl/spectest/Makefile

@@ -2,7 +2,7 @@
 
 
 tests:
-	GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/ethereum/consensus-spec-tests
+	GIT_LFS_SKIP_SMUDGE=1 GIT_CLONE_PROTECTION_ACTIVE=false git clone https://github.com/ethereum/consensus-spec-tests
 	cd consensus-spec-tests && git checkout 080c96fbbf3be58e75947debfeb9ba3b2b7c9748 && git lfs pull --exclude=tests/general,tests/minimal && cd ..
 	mv consensus-spec-tests/tests .
 	rm -rf consensus-spec-tests

cmd/integration/commands/stages.go

@@ -1621,6 +1621,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig,
 		chainConfig,
 		genesisBlock,
 		chainConfig.ChainID.Uint64(),
+		logger,
 	)
 
 	maxBlockBroadcastPeers := func(header *types.Header) uint { return 0 }

cmd/snapshots/manifest/manifest.go

@@ -12,12 +12,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/urfave/cli/v2"
+
 	"github.com/ledgerwatch/erigon-lib/downloader"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
 	"github.com/ledgerwatch/erigon/cmd/snapshots/sync"
 	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/turbo/logging"
-	"github.com/urfave/cli/v2"
 )
 
 var (
@@ -286,7 +287,7 @@ func verifyManifest(ctx context.Context, srcSession *downloader.RCloneSession, v
 	var extra string
 
 	if len(manifestFiles) != 0 {
-		files := make([]string, len(manifestFiles))
+		files := make([]string, 0, len(manifestFiles))
 
 		for file := range manifestFiles {
 			files = append(files, file)
@@ -296,7 +297,7 @@ func verifyManifest(ctx context.Context, srcSession *downloader.RCloneSession, v
 	}
 
 	if len(dirFiles) != 0 {
-		files := make([]string, len(dirFiles))
+		files := make([]string, 0, len(dirFiles))
 
 		for file := range dirFiles {
 			files = append(files, file)

cmd/utils/flags.go

@@ -65,6 +65,7 @@ import (
 	"github.com/ledgerwatch/erigon/p2p/nat"
 	"github.com/ledgerwatch/erigon/p2p/netutil"
 	"github.com/ledgerwatch/erigon/params"
+	borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype"
 	"github.com/ledgerwatch/erigon/rpc/rpccfg"
 	"github.com/ledgerwatch/erigon/turbo/logging"
 )
@@ -627,11 +628,12 @@ var (
 	}
 	NoDiscoverFlag = cli.BoolFlag{
 		Name:  "nodiscover",
-		Usage: "Disables the peer discovery mechanism (manual peer addition)",
+		Usage: "Disables the v4 peer discovery mechanism (manual peer addition). Refer to --v5disc to configure v5 discovery protocol",
 	}
 	DiscoveryV5Flag = cli.BoolFlag{
 		Name:  "v5disc",
-		Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism",
+		Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (enabled by default). Use --v5disc=false to disable it",
+		Value: true,
 	}
 	NetrestrictFlag = cli.StringFlag{
 		Name:  "netrestrict",
@@ -1076,9 +1078,14 @@ var (
 		Usage: "Diagnostics HTTP server listening port",
 		Value: 6060,
 	}
+	DiagSpeedTestFlag = cli.BoolFlag{
+		Name:  "diagnostics.speedtest",
+		Usage: "Enable speed test",
+		Value: false,
+	}
 )
 
-var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag, &DiagDisabledFlag, &DiagEndpointAddrFlag, &DiagEndpointPortFlag}
+var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag, &DiagDisabledFlag, &DiagEndpointAddrFlag, &DiagEndpointPortFlag, &DiagSpeedTestFlag}
 
 var DiagnosticsFlags = []cli.Flag{&DiagnosticsURLFlag, &DiagnosticsURLFlag, &DiagnosticsSessionsFlag}
 
@@ -1658,6 +1665,7 @@ func setBorConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 	cfg.WithoutHeimdall = ctx.Bool(WithoutHeimdallFlag.Name)
 	cfg.WithHeimdallMilestones = ctx.Bool(WithHeimdallMilestones.Name)
 	cfg.WithHeimdallWaypointRecording = ctx.Bool(WithHeimdallWaypoints.Name)
+	borsnaptype.RecordWayPoints(cfg.WithHeimdallWaypointRecording)
 	cfg.PolygonSync = ctx.Bool(PolygonSyncFlag.Name)
 }
 

core/blockchain.go

@@ -217,7 +217,7 @@ func logReceipts(receipts types.Receipts, txns types.Transactions, cc *chain.Con
 		return
 	}
 
-	marshalled := make([]map[string]interface{}, len(receipts))
+	marshalled := make([]map[string]interface{}, 0, len(receipts))
 	for i, receipt := range receipts {
 		txn := txns[i]
 		marshalled = append(marshalled, ethutils.MarshalReceipt(receipt, txn, cc, header, txn.Hash(), true))

core/rawdb/accessors_chain.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"encoding/binary"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"math/big"
@@ -43,7 +42,6 @@ import (
 
 	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/ethdb/cbor"
-	"github.com/ledgerwatch/erigon/polygon/heimdall"
 	"github.com/ledgerwatch/erigon/rlp"
 )
 
@@ -1097,116 +1095,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) error {
 	return nil
 }
 
-// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
-// keeps genesis in db: [1, to)
-// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
-// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
-func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
-	c, err := tx.Cursor(kv.BorEventNums)
-	if err != nil {
-		return err
-	}
-	defer c.Close()
-	var blockNumBytes [8]byte
-	binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
-	k, v, err := c.Seek(blockNumBytes[:])
-	if err != nil {
-		return err
-	}
-	var eventIdTo uint64 = math.MaxUint64
-	if k != nil {
-		eventIdTo = binary.BigEndian.Uint64(v)
-	}
-	c1, err := tx.RwCursor(kv.BorEvents)
-	if err != nil {
-		return err
-	}
-	defer c1.Close()
-	counter := blocksDeleteLimit
-	for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
-		eventId := binary.BigEndian.Uint64(k)
-		if eventId >= eventIdTo {
-			break
-		}
-		if err = c1.DeleteCurrent(); err != nil {
-			return err
-		}
-		counter--
-	}
-	if err != nil {
-		return err
-	}
-	firstSpanToKeep := SpanIdAt(blockTo)
-	c2, err := tx.RwCursor(kv.BorSpans)
-	if err != nil {
-		return err
-	}
-	defer c2.Close()
-	counter = blocksDeleteLimit
-	for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
-		spanId := binary.BigEndian.Uint64(k)
-		if spanId >= firstSpanToKeep {
-			break
-		}
-		if err = c2.DeleteCurrent(); err != nil {
-			return err
-		}
-		counter--
-	}
-
-	checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
-	if err != nil {
-		return err
-	}
-
-	defer checkpointCursor.Close()
-	lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)
-
-	if err != nil {
-		return err
-	}
-
-	var checkpointIdBytes [8]byte
-	binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
-	for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
-		if err = checkpointCursor.DeleteCurrent(); err != nil {
-			return err
-		}
-	}
-
-	milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
-
-	if err != nil {
-		return err
-	}
-
-	defer milestoneCursor.Close()
-
-	var lastMilestoneToRemove heimdall.MilestoneId
-
-	for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
-		lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))
-
-		if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
-			return err
-		} else {
-			if blockCount == blocksDeleteLimit-1 {
-				return nil
-			}
-		}
-	}
-
-	var milestoneIdBytes [8]byte
-	binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
-	for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
-		if err = milestoneCursor.DeleteCurrent(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func TruncateCanonicalChain(ctx context.Context, db kv.RwTx, from uint64) error {
 	return db.ForEach(kv.HeaderCanonical, hexutility.EncodeTs(from), func(k, _ []byte) error {
 		return db.Delete(kv.HeaderCanonical, k)

core/rawdb/blockio/block_writer.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
 	"github.com/ledgerwatch/erigon/core/rawdb"
+	"github.com/ledgerwatch/erigon/polygon/bor/bordb"
 	"github.com/ledgerwatch/log/v3"
 )
 
@@ -116,5 +117,5 @@ func (w *BlockWriter) PruneBlocks(ctx context.Context, tx kv.RwTx, blockTo uint6
 // doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
 // doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
 func (w *BlockWriter) PruneBorBlocks(ctx context.Context, tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
-	return rawdb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
+	return bordb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
 }

core/vm/evm.go

@@ -227,6 +227,12 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp
 		v := value
 		if typ == STATICCALL {
 			v = nil
+		} else if typ == DELEGATECALL {
+			// NOTE: caller must, at all times be a contract. It should never happen
+			// that caller is something other than a Contract.
+			parent := caller.(*Contract)
+			// DELEGATECALL inherits value from parent call
+			v = parent.value
 		}
 		if depth == 0 {
 			evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code)

diagnostics/logs.go

@@ -70,7 +70,7 @@ func writeLogsList(w http.ResponseWriter, dirPath string) {
 		Size int64  `json:"size"`
 	}
 
-	files := make([]file, len(infos))
+	files := make([]file, 0, len(infos))
 
 	for _, fileInfo := range infos {
 		files = append(files, file{Name: fileInfo.Name(), Size: fileInfo.Size()})