op-node: EIP-4844 support

Original rebased prototype by proto, plus changes from Roberto:
- encapsulate data <-> blob conversion code, add unit tests
- update 4844 code to be compatible with latest beacon node api
- remove stray file, include one more invalid blob test
- appropriate fee bumping & blob fee estimation for blob transactions
- misc other improvements

Author: protolambda

op-batcher/batcher/driver.go

@@ -315,7 +315,7 @@ func (l *BatchSubmitter) publishTxToL1(ctx context.Context, queue *txmgr.Queue[t
 	// send all available transactions
 	l1tip, err := l.l1Tip(ctx)
 	if err != nil {
-		l.Log.Error("Failed to query L1 tip", "error", err)
+		l.Log.Error("Failed to query L1 tip", "err", err)
 		return err
 	}
 	l.recordL1Tip(l1tip)
@@ -334,22 +334,37 @@ func (l *BatchSubmitter) publishTxToL1(ctx context.Context, queue *txmgr.Queue[t
 	return nil
 }
 
-// sendTransaction creates & submits a transaction to the batch inbox address with the given `data`.
+// sendTransaction creates & submits a transaction to the batch inbox address with the given `txData`.
 // It currently uses the underlying `txmgr` to handle transaction sending & price management.
 // This is a blocking method. It should not be called concurrently.
 func (l *BatchSubmitter) sendTransaction(txdata txData, queue *txmgr.Queue[txData], receiptsCh chan txmgr.TxReceipt[txData]) {
 	// Do the gas estimation offline. A value of 0 will cause the [txmgr] to estimate the gas limit.
 	data := txdata.Bytes()
+
+	var blobs []*eth.Blob
+	if l.RollupConfig.BlobsEnabledL1Timestamp != nil && *l.RollupConfig.BlobsEnabledL1Timestamp <= uint64(time.Now().Unix()) {
+		var b eth.Blob
+		if err := b.FromData(data); err != nil {
+			l.Log.Error("data could not be converted to blob", "err", err)
+			return
+		}
+		blobs = append(blobs, &b)
+
+		// no calldata
+		data = []byte{}
+	}
+
 	intrinsicGas, err := core.IntrinsicGas(data, nil, false, true, true, false)
 	if err != nil {
-		l.Log.Error("Failed to calculate intrinsic gas", "error", err)
+		l.Log.Error("Failed to calculate intrinsic gas", "err", err)
 		return
 	}
 
 	candidate := txmgr.TxCandidate{
 		To:       &l.RollupConfig.BatchInboxAddress,
 		TxData:   data,
 		GasLimit: intrinsicGas,
+		Blobs:    blobs,
 	}
 	queue.Send(txdata, candidate, receiptsCh)
 }

op-chain-ops/genesis/config.go

@@ -230,6 +230,9 @@ type DeployConfig struct {
 
 	// When Cancun activates. Relative to L1 genesis.
 	L1CancunTimeOffset *uint64 `json:"l1CancunTimeOffset,omitempty"`
+
+	// When 4844 blob-tx functionality for rollup DA actives. Relative to L2 genesis.
+	L2BlobsUpgradeTimeOffset *uint64 `json:"l2BlobsUpgradeTimeOffset,omitempty"`
 }
 
 // Copy will deeply copy the DeployConfig. This does a JSON roundtrip to copy
@@ -522,6 +525,17 @@ func (d *DeployConfig) InteropTime(genesisTime uint64) *uint64 {
 	return &v
 }
 
+func (d *DeployConfig) BlobsUpgradeTime(genesisTime uint64) *uint64 {
+	if d.L2BlobsUpgradeTimeOffset == nil {
+		return nil
+	}
+	v := uint64(0)
+	if offset := *d.L2BlobsUpgradeTimeOffset; offset > 0 {
+		v = genesisTime + uint64(offset)
+	}
+	return &v
+}
+
 // RollupConfig converts a DeployConfig to a rollup.Config
 func (d *DeployConfig) RollupConfig(l1StartBlock *types.Block, l2GenesisBlockHash common.Hash, l2GenesisBlockNumber uint64) (*rollup.Config, error) {
 	if d.OptimismPortalProxy == (common.Address{}) {
@@ -564,6 +578,8 @@ func (d *DeployConfig) RollupConfig(l1StartBlock *types.Block, l2GenesisBlockHas
 		EclipseTime:            d.EclipseTime(l1StartBlock.Time()),
 		FjordTime:              d.FjordTime(l1StartBlock.Time()),
 		InteropTime:            d.InteropTime(l1StartBlock.Time()),
+		// 4844 blobs usage activation for rollup DA
+		BlobsEnabledL1Timestamp: d.BlobsUpgradeTime(l1StartBlock.Time()),
 	}, nil
 }
 

op-e2e/actions/l2_engine.go

@@ -3,12 +3,11 @@ package actions
 import (
 	"errors"
 
-	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
-	"github.com/ethereum-optimism/optimism/op-program/client/l2/engineapi"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/txpool/blobpool"
 	"github.com/ethereum/go-ethereum/core/types"
 	geth "github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
@@ -20,9 +19,11 @@ import (
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
+	"github.com/ethereum-optimism/optimism/op-program/client/l2/engineapi"
 	"github.com/ethereum-optimism/optimism/op-service/client"
-	"github.com/ethereum-optimism/optimism/op-service/eth"
+	opeth "github.com/ethereum-optimism/optimism/op-service/eth"
 	"github.com/ethereum-optimism/optimism/op-service/sources"
 	"github.com/ethereum-optimism/optimism/op-service/testutils"
 )
@@ -48,7 +49,7 @@ type L2Engine struct {
 
 type EngineOption func(ethCfg *ethconfig.Config, nodeCfg *node.Config) error
 
-func NewL2Engine(t Testing, log log.Logger, genesis *core.Genesis, rollupGenesisL1 eth.BlockID, jwtPath string, options ...EngineOption) *L2Engine {
+func NewL2Engine(t Testing, log log.Logger, genesis *core.Genesis, rollupGenesisL1 opeth.BlockID, jwtPath string, options ...EngineOption) *L2Engine {
 	n, ethBackend, apiBackend := newBackend(t, genesis, jwtPath, options)
 	engineApi := engineapi.NewL2EngineAPI(log, apiBackend)
 	chain := ethBackend.BlockChain()
@@ -59,7 +60,7 @@ func NewL2Engine(t Testing, log log.Logger, genesis *core.Genesis, rollupGenesis
 		eth:  ethBackend,
 		rollupGenesis: &rollup.Genesis{
 			L1:     rollupGenesisL1,
-			L2:     eth.BlockID{Hash: genesisBlock.Hash(), Number: genesisBlock.NumberU64()},
+			L2:     opeth.BlockID{Hash: genesisBlock.Hash(), Number: genesisBlock.NumberU64()},
 			L2Time: genesis.Timestamp,
 		},
 		l2Chain:   chain,
@@ -84,6 +85,11 @@ func newBackend(t e2eutils.TestingBase, genesis *core.Genesis, jwtPath string, o
 	ethCfg := &ethconfig.Config{
 		NetworkId: genesis.Config.ChainID.Uint64(),
 		Genesis:   genesis,
+		BlobPool: blobpool.Config{
+			Datadir:   t.TempDir(),
+			Datacap:   blobpool.DefaultConfig.Datacap,
+			PriceBump: blobpool.DefaultConfig.PriceBump,
+		},
 	}
 	nodeCfg := &node.Config{
 		Name:        "l2-geth",

op-e2e/actions/l2_verifier.go

@@ -59,7 +59,8 @@ type L2API interface {
 
 func NewL2Verifier(t Testing, log log.Logger, l1 derive.L1Fetcher, eng L2API, cfg *rollup.Config, syncCfg *sync.Config) *L2Verifier {
 	metrics := &testutils.TestDerivationMetrics{}
-	pipeline := derive.NewDerivationPipeline(log, cfg, l1, eng, metrics, syncCfg)
+	// TODO blob testing
+	pipeline := derive.NewDerivationPipeline(log, cfg, l1, nil, eng, metrics, syncCfg)
 	pipeline.Reset()
 
 	rollupNode := &L2Verifier{

op-e2e/e2eutils/geth/geth.go

@@ -83,7 +83,7 @@ func defaultNodeConfig(name string, jwtPath string) *node.Config {
 type GethOption func(ethCfg *ethconfig.Config, nodeCfg *node.Config) error
 
 // InitL2 inits a L2 geth node.
-func InitL2(name string, l2ChainID *big.Int, genesis *core.Genesis, jwtPath string, opts ...GethOption) (*node.Node, *eth.Ethereum, error) {
+func InitL2(name string, l2ChainID *big.Int, genesis *core.Genesis, jwtPath string, blobPoolDir string, opts ...GethOption) (*node.Node, *eth.Ethereum, error) {
 	ethConfig := &ethconfig.Config{
 		NetworkId: l2ChainID.Uint64(),
 		Genesis:   genesis,
@@ -96,6 +96,11 @@ func InitL2(name string, l2ChainID *big.Int, genesis *core.Genesis, jwtPath stri
 			Recommit:          0,
 			NewPayloadTimeout: 0,
 		},
+		BlobPool: blobpool.Config{
+			Datadir:   blobPoolDir,
+			Datacap:   blobpool.DefaultConfig.Datacap,
+			PriceBump: blobpool.DefaultConfig.PriceBump,
+		},
 	}
 	nodeConfig := defaultNodeConfig(fmt.Sprintf("l2-geth-%v", name), jwtPath)
 	return createGethNode(true, nodeConfig, ethConfig, opts...)

op-e2e/eip4844_test.go

@@ -0,0 +1,110 @@
+package op_e2e
+
+import (
+	"context"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/op-service/client"
+	"github.com/ethereum-optimism/optimism/op-service/sources"
+	"github.com/ethereum-optimism/optimism/op-service/testlog"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// TestSystem4844E2E runs the SystemE2E test with 4844 enabled on L1,
+// and active on the rollup in the op-batcher and verifier.
+func TestSystem4844E2E(t *testing.T) {
+	InitParallel(t)
+
+	cfg := DefaultSystemConfig(t)
+	genesisActivation := uint64(0)
+	cfg.DeployConfig.L1CancunTimeOffset = &genesisActivation
+	cfg.DeployConfig.L2BlobsUpgradeTimeOffset = &genesisActivation
+
+	sys, err := cfg.Start(t)
+	require.Nil(t, err, "Error starting up system")
+	defer sys.Close()
+
+	log := testlog.Logger(t, log.LvlInfo)
+	log.Info("genesis", "l2", sys.RollupConfig.Genesis.L2, "l1", sys.RollupConfig.Genesis.L1, "l2_time", sys.RollupConfig.Genesis.L2Time)
+
+	l1Client := sys.Clients["l1"]
+	l2Seq := sys.Clients["sequencer"]
+	l2Verif := sys.Clients["verifier"]
+
+	// Transactor Account
+	ethPrivKey := cfg.Secrets.Alice
+
+	// Send Transaction & wait for success
+	fromAddr := cfg.Secrets.Addresses().Alice
+	log.Info("alice", "addr", fromAddr)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	startBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
+	require.Nil(t, err)
+
+	// Send deposit transaction
+	opts, err := bind.NewKeyedTransactorWithChainID(ethPrivKey, cfg.L1ChainIDBig())
+	require.Nil(t, err)
+	mintAmount := big.NewInt(1_000_000_000_000)
+	opts.Value = mintAmount
+	SendDepositTx(t, cfg, l1Client, l2Verif, opts, func(l2Opts *DepositTxOpts) {})
+
+	// Confirm balance
+	ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	endBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
+	require.Nil(t, err)
+
+	diff := new(big.Int)
+	diff = diff.Sub(endBalance, startBalance)
+	require.Equal(t, mintAmount, diff, "Did not get expected balance change")
+
+	// Submit TX to L2 sequencer node
+	receipt := SendL2Tx(t, cfg, l2Seq, ethPrivKey, func(opts *TxOpts) {
+		opts.Value = big.NewInt(1_000_000_000)
+		opts.Nonce = 1 // Already have deposit
+		opts.ToAddr = &common.Address{0xff, 0xff}
+		opts.VerifyOnClients(l2Verif)
+	})
+
+	// Verify blocks match after batch submission on verifiers and sequencers
+	verifBlock, err := l2Verif.BlockByNumber(context.Background(), receipt.BlockNumber)
+	require.Nil(t, err)
+	seqBlock, err := l2Seq.BlockByNumber(context.Background(), receipt.BlockNumber)
+	require.Nil(t, err)
+	require.Equal(t, verifBlock.NumberU64(), seqBlock.NumberU64(), "Verifier and sequencer blocks not the same after including a batch tx")
+	require.Equal(t, verifBlock.ParentHash(), seqBlock.ParentHash(), "Verifier and sequencer blocks parent hashes not the same after including a batch tx")
+	require.Equal(t, verifBlock.Hash(), seqBlock.Hash(), "Verifier and sequencer blocks not the same after including a batch tx")
+
+	rollupRPCClient, err := rpc.DialContext(context.Background(), sys.RollupNodes["sequencer"].HTTPEndpoint())
+	require.Nil(t, err)
+	rollupClient := sources.NewRollupClient(client.NewBaseRPCClient(rollupRPCClient))
+	// basic check that sync status works
+	seqStatus, err := rollupClient.SyncStatus(context.Background())
+	require.Nil(t, err)
+	require.LessOrEqual(t, seqBlock.NumberU64(), seqStatus.UnsafeL2.Number)
+	// basic check that version endpoint works
+	seqVersion, err := rollupClient.Version(context.Background())
+	require.Nil(t, err)
+	require.NotEqual(t, "", seqVersion)
+
+	// quick check that the batch submitter works
+	for i := 0; i < 10; i++ {
+		// wait for chain to be marked as "safe" (i.e. confirm batch-submission works)
+		stat, err := rollupClient.SyncStatus(context.Background())
+		require.NoError(t, err)
+		if stat.SafeL2.Number > 0 {
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatal("expected L2 to be batch-submitted and labeled as safe")
+}

op-e2e/op_geth.go

@@ -76,7 +76,7 @@ func NewOpGeth(t *testing.T, ctx context.Context, cfg *SystemConfig) (*OpGeth, e
 
 	var node EthInstance
 	if cfg.ExternalL2Shim == "" {
-		gethNode, _, err := geth.InitL2("l2", big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath)
+		gethNode, _, err := geth.InitL2("l2", big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, t.TempDir())
 		require.Nil(t, err)
 		require.Nil(t, gethNode.Start())
 		node = gethNode

op-e2e/setup.go

@@ -494,6 +494,8 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 			FjordTime:               cfg.DeployConfig.FjordTime(uint64(cfg.DeployConfig.L1GenesisBlockTimestamp)),
 			InteropTime:             cfg.DeployConfig.InteropTime(uint64(cfg.DeployConfig.L1GenesisBlockTimestamp)),
 			ProtocolVersionsAddress: cfg.L1Deployments.ProtocolVersionsProxy,
+			// 4844
+			BlobsEnabledL1Timestamp: cfg.DeployConfig.BlobsUpgradeTime(uint64(cfg.DeployConfig.L1GenesisBlockTimestamp)),
 		}
 	}
 	defaultConfig := makeRollupConfig()
@@ -509,8 +511,8 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 		_ = bcn.Close()
 	})
 	require.NoError(t, bcn.Start("127.0.0.1:0"))
-	beaconApiAddr := bcn.BeaconAddr()
-	require.NotEmpty(t, beaconApiAddr, "beacon API listener must be up")
+	sys.L1BeaconAPIAddr = bcn.BeaconAddr()
+	require.NotEmpty(t, sys.L1BeaconAPIAddr, "beacon API listener must be up")
 
 	// Initialize nodes
 	l1Node, l1Backend, err := geth.InitL1(cfg.DeployConfig.L1ChainID, cfg.DeployConfig.L1BlockTime, l1Genesis, c,
@@ -529,8 +531,9 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 
 	for name := range cfg.Nodes {
 		var ethClient EthInstance
+		blobPoolPath := path.Join(cfg.BlobsPath, name)
 		if cfg.ExternalL2Shim == "" {
-			node, backend, err := geth.InitL2(name, big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, cfg.GethOptions[name]...)
+			node, backend, err := geth.InitL2(name, big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, blobPoolPath, cfg.GethOptions[name]...)
 			if err != nil {
 				return nil, err
 			}
@@ -548,10 +551,11 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 				t.Skip("External L2 nodes do not support configuration through GethOptions")
 			}
 			ethClient = (&ExternalRunner{
-				Name:    name,
-				BinPath: cfg.ExternalL2Shim,
-				Genesis: l2Genesis,
-				JWTPath: cfg.JWTFilePath,
+				Name:         name,
+				BinPath:      cfg.ExternalL2Shim,
+				Genesis:      l2Genesis,
+				JWTPath:      cfg.JWTFilePath,
+				BlobPoolPath: blobPoolPath,
 			}).Run(t)
 		}
 		sys.EthInstances[name] = ethClient
@@ -562,7 +566,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 	// of only websockets (which are required for external eth client tests).
 	for name, rollupCfg := range cfg.Nodes {
 		configureL1(rollupCfg, sys.EthInstances["l1"])
-		configureL2(rollupCfg, sys.EthInstances[name], cfg.JWTSecret)
+		configureL2(rollupCfg, sys.EthInstances[name], cfg.JWTSecret, sys.L1BeaconAPIAddr)
 
 		rollupCfg.L2Sync = &rollupNode.PreparedL2SyncEndpoint{
 			Client:   nil,
@@ -870,7 +874,7 @@ type WSOrHTTPEndpoint interface {
 	HTTPAuthEndpoint() string
 }
 
-func configureL2(rollupNodeCfg *rollupNode.Config, l2Node WSOrHTTPEndpoint, jwtSecret [32]byte) {
+func configureL2(rollupNodeCfg *rollupNode.Config, l2Node WSOrHTTPEndpoint, jwtSecret [32]byte, l1BeaconAPIAddr string) {
 	l2EndpointConfig := l2Node.WSAuthEndpoint()
 	if UseHTTP() {
 		l2EndpointConfig = l2Node.HTTPAuthEndpoint()
@@ -880,6 +884,7 @@ func configureL2(rollupNodeCfg *rollupNode.Config, l2Node WSOrHTTPEndpoint, jwtS
 		L2EngineAddr:      l2EndpointConfig,
 		L2EngineJWTSecret: jwtSecret,
 	}
+	rollupNodeCfg.Beacon = &rollupNode.L1BeaconEndpointConfig{BeaconAddr: l1BeaconAPIAddr}
 }
 
 func (cfg SystemConfig) L1ChainIDBig() *big.Int {

op-e2e/system_test.go

@@ -780,11 +780,11 @@ func TestSystemP2PAltSync(t *testing.T) {
 		},
 	}
 	configureL1(syncNodeCfg, sys.EthInstances["l1"])
-	syncerL2Engine, _, err := geth.InitL2("syncer", big.NewInt(int64(cfg.DeployConfig.L2ChainID)), sys.L2GenesisCfg, cfg.JWTFilePath)
+	syncerL2Engine, _, err := geth.InitL2("syncer", big.NewInt(int64(cfg.DeployConfig.L2ChainID)), sys.L2GenesisCfg, cfg.JWTFilePath, t.TempDir())
 	require.NoError(t, err)
 	require.NoError(t, syncerL2Engine.Start())
 
-	configureL2(syncNodeCfg, syncerL2Engine, cfg.JWTSecret)
+	configureL2(syncNodeCfg, syncerL2Engine, cfg.JWTSecret, sys.L1BeaconAPIAddr)
 
 	syncerNode, err := rollupNode.New(context.Background(), syncNodeCfg, cfg.Loggers["syncer"], snapLog, "", metrics.NewMetrics(""))
 	require.NoError(t, err)