Merge pull request #3116 from ethereum/dev

Release v1.3.0-alpha.1 -- capella and 4844 prelease

Author: djrtwo

Makefile

@@ -41,6 +41,8 @@ CURRENT_DIR = ${CURDIR}
 LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini
 GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt
 
+SCRIPTS_DIR = ${CURRENT_DIR}/scripts
+
 export DAPP_SKIP_BUILD:=1
 export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)
 export DAPP_LIB:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)/lib
@@ -195,6 +197,14 @@ $(TEST_VECTOR_DIR):
 $(TEST_VECTOR_DIR)/:
 	$(info ignoring duplicate tests dir)
 
+gen_kzg_setups:
+	cd $(SCRIPTS_DIR); \
+	if ! test -d venv; then python3 -m venv venv; fi; \
+	. venv/bin/activate; \
+	pip3 install -r requirements.txt; \
+	python3 ./gen_kzg_trusted_setups.py --secret=1337 --length=4 --output-dir ${CURRENT_DIR}/presets/minimal/trusted_setups; \
+	python3 ./gen_kzg_trusted_setups.py --secret=1337 --length=4096 --output-dir ${CURRENT_DIR}/presets/mainnet/trusted_setups
+
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
 gen_%: $(TEST_VECTOR_DIR)

README.md

@@ -2,7 +2,7 @@
 
 [![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-To learn more about proof-of-stake and sharding, see the [PoS FAQ](https://eth.wiki/en/concepts/proof-of-stake-faqs), [sharding FAQ](https://eth.wiki/sharding/Sharding-FAQs) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
+To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
 
 This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests.
 

presets/mainnet/capella.yaml

@@ -1,23 +1,11 @@
 # Mainnet preset - Capella
 
 # Misc
-# ---------------------------------------------------------------
-# 2**8 (= 256) withdrawals
-MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 256
-
-
-# State list lengths
-# ---------------------------------------------------------------
-# 2**40 (= 1,099,511,627,776) withdrawals
-WITHDRAWAL_QUEUE_LIMIT: 1099511627776
-
-
 # Max operations per block
 # ---------------------------------------------------------------
 # 2**4 (= 16)
 MAX_BLS_TO_EXECUTION_CHANGES: 16
 
-
 # Execution
 # ---------------------------------------------------------------
 # 2**4 (= 16) withdrawals

presets/mainnet/trusted_setups/testing_trusted_setups.json

@@ -1,17 +1,5 @@
 # Minimal preset - Capella
 
-# Misc
-# ---------------------------------------------------------------
-# [customized] 16 for more interesting tests at low validator count
-MAX_PARTIAL_WITHDRAWALS_PER_EPOCH: 16
-
-
-# State list lengths
-# ---------------------------------------------------------------
-# 2**40 (= 1,099,511,627,776) withdrawals
-WITHDRAWAL_QUEUE_LIMIT: 1099511627776
-
-
 # Max operations per block
 # ---------------------------------------------------------------
 # 2**4 (= 16)
@@ -20,5 +8,5 @@ MAX_BLS_TO_EXECUTION_CHANGES: 16
 
 # Execution
 # ---------------------------------------------------------------
-# [customized] Lower than MAX_PARTIAL_WITHDRAWALS_PER_EPOCH so not all processed in one block
-MAX_WITHDRAWALS_PER_PAYLOAD: 8
+# [customized] 2**2 (= 4)
+MAX_WITHDRAWALS_PER_PAYLOAD: 4

presets/minimal/capella.yaml

@@ -0,0 +1 @@
+{"setup_G1": ["0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", "0x854262641262cb9e056a8512808ea6864d903dbcad713fd6da8dddfa5ce40d85612c912063ace060ed8c4bf005bab839", "0x86f708eee5ae0cf40be36993e760d9cb3b2371f22db3209947c5d21ea68e55186b30871c50bf11ef29e5248bf42d5678", "0x94f9c0bafb23cbbf34a93a64243e3e0f934b57593651f3464de7dc174468123d9698f1b9dfa22bb5b6eb96eae002f29f"], "setup_G2": ["0x93e02b6052719f607dacd3a088274f65596bd0d09920b61ab5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8", "0x99aca9fb2f7760cecb892bf7262c176b334824f5727f680bba701a33e322cb6667531410dfc7c8e4321a3f0ea8af48cb1436638a2093123f046f0f504cc2a864825542873edbbc5d7ed17af125a4f2cf6433c6f4f61b81173726981dd989761d", "0x88e2e982982bf8231e747e9dfcd14c05bd02623d1332734d2af26246c6869fb56ee6c994843f593178a040495ba61f4a083b0e18110b1d9f5224783d8f9a895e8ee744e87929430e9ba96bd29251cbf61240b256d1525600f3d562894d93d659", "0xa2d33775e3d9e6af0d1b27d389e6c021a578e617a3d6627686db6288d4b3dffd7a847a00f7ef01828b7f42885b660e4204923402aca18fbae74ccd4e9c50dd8c2281b38dc09c022342ed1ac695d53f7081cb21f05fdfc0a3508c04759196fcd3"], "setup_G1_lagrange": ["0x91131b2e3c1e5f0b51df8970e67080032f411571b66d301436c46f25bbfddf9ca16756430dc470bdb0d85b47fedcdbc1", "0x934d35b2a46e169915718b77127b0d4efbacdad7fdde4593af7d21d37ebcb77fe6c8dde6b8a9537854d70ef1f291a585", "0x9410ca1d0342fe7419f02194281df45e1c1ff42fd8b439de5644cc312815c21ddd2e3eeb63fb807cf837e68b76668bd5", "0xb163df7e9baeb60f69b6ee5faa538c3a564b62eb8cde6a3616083c8cb2171eedd583c9143e7e916df59bf27da5e024e8"], "roots_of_unity": [1, 3465144826073652318776269530687742778270252468765361963008, 52435875175126190479447740508185965837690552500527637822603658699938581184512, 52435875175126190475982595682112313518914282969839895044333406231173219221505]}

presets/minimal/trusted_setups/testing_trusted_setups.json

@@ -0,0 +1,36 @@
+import os
+from pathlib import Path
+
+from eth2spec.utils.kzg import (
+    dump_kzg_trusted_setup_files,
+)
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--secret",
+        dest="secret",
+        type=int,
+        required=True,
+        help='the secret of trusted setup',
+    )
+    parser.add_argument(
+        "--length",
+        dest="length",
+        type=int,
+        required=True,
+        help='the length of trusted setup',
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        dest="output_dir",
+        required=True,
+        help='the output directory',
+    )
+    args = parser.parse_args()
+
+    dump_kzg_trusted_setup_files(args.secret, args.length, args.output_dir)

scripts/gen_kzg_trusted_setups.py

@@ -0,0 +1 @@
+../[generator]

scripts/requirements.txt

@@ -7,13 +7,14 @@
 import re
 import string
 import textwrap
-from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar
+from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar, Tuple
 from abc import ABC, abstractmethod
 import ast
 import subprocess
 import sys
 import copy
 from collections import OrderedDict
+import json
 
 
 # NOTE: have to programmatically include third-party dependencies in `setup.py`.
@@ -121,7 +122,7 @@ def _get_self_type_from_source(source: str) -> Optional[str]:
     return args[0].annotation.id
 
 
-def _get_class_info_from_source(source: str) -> (str, Optional[str]):
+def _get_class_info_from_source(source: str) -> Tuple[str, Optional[str]]:
     class_def = ast.parse(source).body[0]
     base = class_def.bases[0]
     if isinstance(base, ast.Name):
@@ -140,6 +141,28 @@ def _is_constant_id(name: str) -> bool:
     return all(map(lambda c: c in string.ascii_uppercase + '_' + string.digits, name[1:]))
 
 
+def _load_kzg_trusted_setups(preset_name):
+    """
+    [TODO] it's not the final mainnet trusted setup.
+    We will update it after the KZG ceremony.
+    """
+    file_path = str(Path(__file__).parent) + '/presets/' + preset_name + '/trusted_setups/testing_trusted_setups.json'
+
+    with open(file_path, 'r') as f:
+        json_data = json.load(f)
+
+    trusted_setup_G1 = json_data['setup_G1']
+    trusted_setup_G2 = json_data['setup_G2']
+    trusted_setup_G1_lagrange = json_data['setup_G1_lagrange']
+    roots_of_unity = json_data['roots_of_unity']
+
+    return trusted_setup_G1, trusted_setup_G2, trusted_setup_G1_lagrange, roots_of_unity
+
+ALL_KZG_SETUPS = {
+    'minimal': _load_kzg_trusted_setups('minimal'),
+    'mainnet': _load_kzg_trusted_setups('mainnet')
+}
+
 ETH2_SPEC_COMMENT_PREFIX = "eth2spec:"
 
 
@@ -167,7 +190,16 @@ def _parse_value(name: str, typed_value: str, type_hint: Optional[str]=None) ->
     return VariableDefinition(type_name=type_name, value=typed_value[i+1:-1], comment=comment, type_hint=type_hint)
 
 
-def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> SpecObject:
+def _update_constant_vars_with_kzg_setups(constant_vars, preset_name):
+    comment = "noqa: E501"
+    kzg_setups = ALL_KZG_SETUPS[preset_name]
+    constant_vars['KZG_SETUP_G1'] = VariableDefinition(constant_vars['KZG_SETUP_G1'].value, str(kzg_setups[0]), comment, None)
+    constant_vars['KZG_SETUP_G2'] = VariableDefinition(constant_vars['KZG_SETUP_G2'].value, str(kzg_setups[1]), comment, None)
+    constant_vars['KZG_SETUP_LAGRANGE'] = VariableDefinition(constant_vars['KZG_SETUP_LAGRANGE'].value, str(kzg_setups[2]), comment, None)
+    constant_vars['ROOTS_OF_UNITY'] = VariableDefinition(constant_vars['ROOTS_OF_UNITY'].value, str(kzg_setups[3]), comment, None)
+
+
+def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], preset_name=str) -> SpecObject:
     functions: Dict[str, str] = {}
     protocols: Dict[str, ProtocolDefinition] = {}
     constant_vars: Dict[str, VariableDefinition] = {}
@@ -232,7 +264,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
 
                     if not _is_constant_id(name):
                         # Check for short type declarations
-                        if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List")):
+                        if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")):
                             custom_types[name] = value
                         continue
 
@@ -256,6 +288,10 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
             if comment == "skip":
                 should_skip = True
 
+    # Load KZG trusted setup from files
+    if any('KZG_SETUP' in name for name in constant_vars):
+        _update_constant_vars_with_kzg_setups(constant_vars, preset_name)
+
     return SpecObject(
         functions=functions,
         protocols=protocols,
@@ -582,15 +618,13 @@ def imports(cls, preset_name: str):
 #
 # EIP4844SpecBuilder
 #
-class EIP4844SpecBuilder(BellatrixSpecBuilder):
+class EIP4844SpecBuilder(CapellaSpecBuilder):
     fork: str = EIP4844
 
     @classmethod
     def imports(cls, preset_name: str):
         return super().imports(preset_name) + f'''
-from eth2spec.utils import kzg
-from eth2spec.bellatrix import {preset_name} as bellatrix
-from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize
+from eth2spec.capella import {preset_name} as capella
 '''
 
 
@@ -603,33 +637,45 @@ def preparations(cls):
     @classmethod
     def sundry_functions(cls) -> str:
         return super().sundry_functions() + '\n\n' + '''
-# TODO: for mainnet, load pre-generated trusted setup file to reduce building time.
-# TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets
-TESTING_FIELD_ELEMENTS_PER_BLOB = 4
-TESTING_SECRET = 1337
-TESTING_KZG_SETUP_G1 = kzg.generate_setup(bls.G1, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
-TESTING_KZG_SETUP_G2 = kzg.generate_setup(bls.G2, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
-TESTING_KZG_SETUP_LAGRANGE = kzg.get_lagrange(TESTING_KZG_SETUP_G1)
+#
+# Temporarily disable Withdrawals functions for EIP4844 testnets
+#
 
-KZG_SETUP_G1 = [bls.G1_to_bytes48(p) for p in TESTING_KZG_SETUP_G1]
-KZG_SETUP_G2 = [bls.G2_to_bytes96(p) for p in TESTING_KZG_SETUP_G2]
-KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE
-ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB)
 
+def no_op(fn):  # type: ignore
+    def wrapper(*args, **kw):  # type: ignore
+        return None
+    return wrapper
+
+
+def get_empty_list_result(fn):  # type: ignore
+    def wrapper(*args, **kw):  # type: ignore
+        return []
+    return wrapper
 
-def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar:
-    pass'''
+
+process_withdrawals = no_op(process_withdrawals)
+process_bls_to_execution_change = no_op(process_bls_to_execution_change)
+get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals)
+
+
+#
+# End
+#
+
+def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> Optional[BlobsSidecar]:
+    return "TEST"'''
 
     @classmethod
     def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
         constants = {
+            'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
             'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
         }
         return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}
 
 
-
 spec_builders = {
     builder.fork: builder
     for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, EIP4844SpecBuilder)
@@ -880,7 +926,7 @@ def _build_spec(preset_name: str, fork: str,
                 source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
     preset = load_preset(preset_files)
     config = load_config(config_file)
-    all_specs = [get_spec(spec, preset, config) for spec in source_files]
+    all_specs = [get_spec(spec, preset, config, preset_name) for spec in source_files]
 
     spec_object = all_specs[0]
     for value in all_specs[1:]:
@@ -967,7 +1013,7 @@ def finalize_options(self):
                     specs/bellatrix/p2p-interface.md
                     sync/optimistic.md
                 """
-            if self.spec_fork == CAPELLA:
+            if self.spec_fork in (CAPELLA, EIP4844):
                 self.md_doc_paths += """
                     specs/capella/beacon-chain.md
                     specs/capella/fork.md
@@ -1131,7 +1177,7 @@ def run(self):
         "pycryptodome==3.15.0",
         "py_ecc==6.0.0",
         "milagro_bls_binding==1.9.0",
-        "remerkleable==0.1.24",
+        "remerkleable==0.1.25",
         RUAMEL_YAML_VERSION,
         "lru-dict==1.1.8",
         MARKO_VERSION,

setup.py

@@ -84,13 +84,13 @@ def create_light_client_update(state: BeaconState,
     header = state.latest_block_header.copy()
     header.state_root = hash_tree_root(state)
     assert hash_tree_root(header) == hash_tree_root(block.message)
-    update_signature_period = compute_sync_committee_period(compute_epoch_at_slot(block.message.slot))
+    update_signature_period = compute_sync_committee_period_at_slot(block.message.slot)
 
     assert attested_state.slot == attested_state.latest_block_header.slot
     attested_header = attested_state.latest_block_header.copy()
     attested_header.state_root = hash_tree_root(attested_state)
     assert hash_tree_root(attested_header) == block.message.parent_root
-    update_attested_period = compute_sync_committee_period(compute_epoch_at_slot(attested_header.slot))
+    update_attested_period = compute_sync_committee_period_at_slot(attested_header.slot)
 
     # `next_sync_committee` is only useful if the message is signed by the current sync committee
     if update_attested_period == update_signature_period:
@@ -133,7 +133,7 @@ def create_light_client_update(state: BeaconState,
 Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods.
 
 - `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.slot`
-- `LightClientUpdate` are only considered if `compute_sync_committee_period(compute_epoch_at_slot(update.attested_header.slot)) == compute_sync_committee_period(compute_epoch_at_slot(update.signature_slot))`
+- `LightClientUpdate` are only considered if `compute_sync_committee_period_at_slot(update.attested_header.slot) == compute_sync_committee_period_at_slot(update.signature_slot)`
 - Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time.
 
 ### `create_light_client_finality_update`