[DRAFT] Generate Hive Test Fixtures

configs/hive.yaml

@@ -0,0 +1,95 @@
+# Hive config
+
+# Extends the hive preset
+PRESET_BASE: 'hive'
+
+# Free-form short name of the network that this configuration applies to - known
+# canonical network names include:
+# * 'mainnet' - there can be only one
+# * 'prater' - testnet
+# Must match the regex: [a-z0-9\-]
+CONFIG_NAME: 'hive'
+
+# Transition
+# ---------------------------------------------------------------
+# Estimated on Sept 15, 2022
+TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000
+# By default, don't use these params
+TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
+TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
+
+
+
+# Genesis
+# ---------------------------------------------------------------
+# `2**14` (= 16,384)
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384
+# Dec 1, 2020, 12pm UTC
+MIN_GENESIS_TIME: 1606824000
+# Hive initial fork version
+GENESIS_FORK_VERSION: 0x0000000a
+# 604800 seconds (7 days)
+GENESIS_DELAY: 604800
+
+
+# Forking
+# ---------------------------------------------------------------
+# Some forks are disabled for now:
+#  - These may be re-assigned to another fork-version later
+#  - Temporarily set to max uint64 value: 2**64 - 1
+
+# Altair
+ALTAIR_FORK_VERSION: 0x0100000a
+ALTAIR_FORK_EPOCH: 74240  # Oct 27, 2021, 10:56:23am UTC
+# Bellatrix
+BELLATRIX_FORK_VERSION: 0x0200000a
+BELLATRIX_FORK_EPOCH: 144896  # Sept 6, 2022, 11:34:47am UTC
+# Capella
+CAPELLA_FORK_VERSION: 0x0300000a
+CAPELLA_FORK_EPOCH: 194048  # April 12, 2023, 10:27:35pm UTC
+# Deneb
+DENEB_FORK_VERSION: 0x0400000a
+DENEB_FORK_EPOCH: 18446744073709551615
+
+
+
+
+# Time parameters
+# ---------------------------------------------------------------
+# 12 seconds
+SECONDS_PER_SLOT: 12
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 14
+# 2**8 (= 256) epochs ~27 hours
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# 2**8 (= 256) epochs ~27 hours
+SHARD_COMMITTEE_PERIOD: 256
+# 2**11 (= 2,048) Eth1 blocks ~8 hours
+ETH1_FOLLOW_DISTANCE: 2048
+
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# 2**2 (= 4)
+MIN_PER_EPOCH_CHURN_LIMIT: 4
+# 2**16 (= 65,536)
+CHURN_LIMIT_QUOTIENT: 65536
+
+
+# Fork choice
+# ---------------------------------------------------------------
+# 40%
+PROPOSER_SCORE_BOOST: 40
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Ethereum PoW Mainnet
+DEPOSIT_CHAIN_ID: 1
+DEPOSIT_NETWORK_ID: 1
+DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa

setup.py

@@ -164,6 +164,10 @@ def _load_kzg_trusted_setups(preset_name):
     'mainnet': _load_kzg_trusted_setups('mainnet')
 }
 
+EQUIVALENT_KZG_SETUPS = {
+    'hive': 'mainnet',
+}
+
 ETH2_SPEC_COMMENT_PREFIX = "eth2spec:"
 
 
@@ -193,6 +197,8 @@ def _parse_value(name: str, typed_value: str, type_hint: Optional[str]=None) ->
 
 def _update_constant_vars_with_kzg_setups(constant_vars, preset_name):
     comment = "noqa: E501"
+    if preset_name in EQUIVALENT_KZG_SETUPS:
+        preset_name = EQUIVALENT_KZG_SETUPS[preset_name]
     kzg_setups = ALL_KZG_SETUPS[preset_name]
     constant_vars['KZG_SETUP_G1'] = VariableDefinition(constant_vars['KZG_SETUP_G1'].value, str(kzg_setups[0]), comment, None)
     constant_vars['KZG_SETUP_G2'] = VariableDefinition(constant_vars['KZG_SETUP_G2'].value, str(kzg_setups[1]), comment, None)
@@ -210,6 +216,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
     ssz_objects: Dict[str, str] = {}
     dataclasses: Dict[str, str] = {}
     custom_types: Dict[str, str] = {}
+    preset_base = config.get('PRESET_BASE')
+    if preset_base is not None:
+        preset_base = preset_base.strip("'")
 
     with open(file_name) as source_file:
         document = gfm.parse(source_file.read())
@@ -291,7 +300,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
     # Load KZG trusted setup from files
     if any('KZG_SETUP' in name for name in constant_vars):
-        _update_constant_vars_with_kzg_setups(constant_vars, preset_name)
+        _update_constant_vars_with_kzg_setups(constant_vars, preset_name if preset_base is None else preset_base)
 
     return SpecObject(
         functions=functions,
@@ -692,6 +701,7 @@ def is_byte_vector(value: str) -> bool:
 
 
 def objects_to_spec(preset_name: str,
+                    preset_base: Optional[str],
                     spec_object: SpecObject,
                     builder: SpecBuilder,
                     ordered_class_objects: Dict[str, str]) -> str:
@@ -744,7 +754,7 @@ def format_config_var(name: str, vardef: VariableDefinition) -> str:
     config_spec += '\n'.join(f'    {k}: {v.type_name if v.type_name is not None else "int"}'
                              for k, v in spec_object.config_vars.items())
     config_spec += '\n\n\nconfig = Configuration(\n'
-    config_spec += f'    PRESET_BASE="{preset_name}",\n'
+    config_spec += f'    PRESET_BASE="{preset_name if preset_base is None else preset_base}",\n'
     config_spec += '\n'.join('    ' + format_config_var(k, v) for k, v in spec_object.config_vars.items())
     config_spec += '\n)\n'
 
@@ -936,8 +946,10 @@ def _build_spec(preset_name: str, fork: str,
     while OrderedDict(new_objects) != OrderedDict(class_objects):
         new_objects = copy.deepcopy(class_objects)
         dependency_order_class_objects(class_objects, spec_object.custom_types)
-
-    return objects_to_spec(preset_name, spec_object, spec_builders[fork], class_objects)
+    preset_base = config.get('PRESET_BASE')
+    if preset_base is not None:
+        preset_base = preset_base.strip("'")
+    return objects_to_spec(preset_name, preset_base, spec_object, spec_builders[fork], class_objects)
 
 
 class BuildTarget(NamedTuple):
@@ -975,6 +987,7 @@ def initialize_options(self):
         self.build_targets = """
                 minimal:presets/minimal:configs/minimal.yaml
                 mainnet:presets/mainnet:configs/mainnet.yaml
+                hive:presets/mainnet:configs/hive.yaml
         """
 
     def finalize_options(self):

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -3,7 +3,7 @@
 from typing import Any, Callable, Dict, Iterable, Optional, List, Union
 
 from eth2spec.utils import bls
-from eth2spec.test.helpers.constants import ALL_PRESETS, TESTGEN_FORKS
+from eth2spec.test.helpers.constants import ALL_PRESETS, HIVE, TESTGEN_FORKS
 from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
 
 from eth2spec.gen_helpers.gen_base import gen_runner
@@ -96,10 +96,13 @@ def cases_fn() -> Iterable[TestCase]:
 def run_state_test_generators(runner_name: str,
                               all_mods: Dict[str, Dict[str, str]],
                               presets: Iterable[PresetBaseName] = ALL_PRESETS,
-                              forks: Iterable[SpecForkName] = TESTGEN_FORKS) -> None:
+                              forks: Iterable[SpecForkName] = TESTGEN_FORKS,
+                              is_hive: bool = False) -> None:
     """
     Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_PRESETS` presets of the given runner.
     """
+    if is_hive:
+        presets = [HIVE]
     for preset_name in presets:
         for fork_name in forks:
             if fork_name in all_mods:

tests/core/pyspec/eth2spec/test/capella/api/test_api.py

@@ -0,0 +1,90 @@
+from eth2spec.test.helpers.hive import (
+    StateID,
+    Eth2BeaconChainRequestBeaconBlocksByRange,
+    EthV1BeaconStatesFinalityCheckpoints,
+    EthV1BeaconStatesFork,
+)
+from eth2spec.test.context import (
+    with_capella_and_later,
+    spec_state_test_with_matching_config,
+    hive_state,
+)
+from eth2spec.test.helpers.state import (
+    state_transition_and_sign_block,
+    next_slot,
+)
+from eth2spec.test.helpers.block import (
+    build_empty_block_for_next_slot,
+)
+from eth2spec.test.helpers.withdrawals import (
+    prepare_expected_withdrawals,
+)
+
+
+def _perform_valid_withdrawal(spec, state):
+    fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals(
+        spec, state, num_partial_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD * 2,
+        num_full_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD * 2)
+
+    next_slot(spec, state)
+    pre_next_withdrawal_index = state.next_withdrawal_index
+
+    expected_withdrawals = spec.get_expected_withdrawals(state)
+
+    pre_state = state.copy()
+
+    # Block 1
+    block = build_empty_block_for_next_slot(spec, state)
+    signed_block_1 = state_transition_and_sign_block(spec, state, block)
+
+    withdrawn_indices = [withdrawal.validator_index for withdrawal in expected_withdrawals]
+    fully_withdrawable_indices = list(set(fully_withdrawable_indices).difference(set(withdrawn_indices)))
+    partial_withdrawals_indices = list(set(partial_withdrawals_indices).difference(set(withdrawn_indices)))
+    assert state.next_withdrawal_index == pre_next_withdrawal_index + spec.MAX_WITHDRAWALS_PER_PAYLOAD
+
+    withdrawn_indices = [withdrawal.validator_index for withdrawal in expected_withdrawals]
+    fully_withdrawable_indices = list(set(fully_withdrawable_indices).difference(set(withdrawn_indices)))
+    partial_withdrawals_indices = list(set(partial_withdrawals_indices).difference(set(withdrawn_indices)))
+    assert state.next_withdrawal_index == pre_next_withdrawal_index + spec.MAX_WITHDRAWALS_PER_PAYLOAD
+
+    return pre_state, signed_block_1, pre_next_withdrawal_index
+
+
+@with_capella_and_later
+@spec_state_test_with_matching_config
+@hive_state()
+def test_debug_beacon_state_v2(spec, state):
+    _, signed_block_1, pre_next_withdrawal_index = (_perform_valid_withdrawal(spec, state))
+
+    # Block 2
+    block = build_empty_block_for_next_slot(spec, state)
+    signed_block_2 = state_transition_and_sign_block(spec, state, block)
+
+    assert state.next_withdrawal_index == pre_next_withdrawal_index + spec.MAX_WITHDRAWALS_PER_PAYLOAD * 2
+    yield 'blocks', [signed_block_1, signed_block_2]
+    yield 'post', state
+
+    yield 'hive', [
+        (
+            EthV1BeaconStatesFinalityCheckpoints(id=StateID.Head(), finalized=False).
+            from_state(state)
+        ),
+        (
+            EthV1BeaconStatesFork(id=StateID.Head(), finalized=False).
+            from_state(state)
+        ),
+        (
+            Eth2BeaconChainRequestBeaconBlocksByRange(
+                start_slot=1,
+                count=2,  # Slot 2 is empty
+                expected_roots=[signed_block_1.hash_tree_root()]
+            )
+        ),
+        (
+            Eth2BeaconChainRequestBeaconBlocksByRange(
+                start_slot=1,
+                count=3,
+                expected_roots=[block.hash_tree_root() for block in [signed_block_1, signed_block_2]],
+            )
+        ),
+    ]

tests/core/pyspec/eth2spec/test/context.py

@@ -3,19 +3,19 @@
 from dataclasses import dataclass
 import importlib
 
-from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal
-from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal
-from eth2spec.bellatrix import mainnet as spec_bellatrix_mainnet, minimal as spec_bellatrix_minimal
-from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal
-from eth2spec.deneb import mainnet as spec_deneb_mainnet, minimal as spec_deneb_minimal
+from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal, hive as spec_phase0_hive
+from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal, hive as spec_altair_hive
+from eth2spec.bellatrix import mainnet as spec_bellatrix_mainnet, minimal as spec_bellatrix_minimal, hive as spec_bellatrix_hive
+from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal, hive as spec_capella_hive
+from eth2spec.deneb import mainnet as spec_deneb_mainnet, minimal as spec_deneb_minimal, hive as spec_deneb_hive
 from eth2spec.eip6110 import mainnet as spec_eip6110_mainnet, minimal as spec_eip6110_minimal
 from eth2spec.utils import bls
 
 from .exceptions import SkippedTest
 from .helpers.constants import (
     PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB,
     EIP6110,
-    MINIMAL, MAINNET,
+    MINIMAL, MAINNET, HIVE,
     ALL_PHASES,
     ALL_FORK_UPGRADES,
 )
@@ -91,6 +91,13 @@ class ForkMeta:
         DENEB: spec_deneb_mainnet,
         EIP6110: spec_eip6110_mainnet,
     },
+    HIVE: {
+        PHASE0: spec_phase0_hive,
+        ALTAIR: spec_altair_hive,
+        BELLATRIX: spec_bellatrix_hive,
+        CAPELLA: spec_capella_hive,
+        DENEB: spec_deneb_hive,
+    },
 }
 
 
@@ -744,3 +751,62 @@ def post_tag(obj):
 
         return wrapper
     return decorator
+
+
+#
+# Hive state modifiers
+#
+
+def hive_state(**decorator_kwargs):
+    def decorator(fn):
+        """
+        Makes necessary changes to the state in order for the client to accept it in hive mode.
+        """
+        def wrapper(*args, **kwargs):
+            if 'state' not in kwargs or 'spec' not in kwargs:
+                raise Exception("hive_state decorator requires state and spec")
+            state = kwargs['state']
+            spec = kwargs['spec']
+
+            # Increase genesis time to min genesis time
+            state.genesis_time = spec.config.MIN_GENESIS_TIME
+            kwargs['state'] = state
+
+            res = fn(*args, **kwargs)
+            if res is not None:
+                yield 'genesis', state
+                yield from res
+
+            # Also yield extra configuration that is for the client in hive mode
+
+            # Time is the next slot after test ends
+            time = state.genesis_time + ((state.slot + 1) * spec.config.SECONDS_PER_SLOT)
+            if "time" in decorator_kwargs:
+                time = decorator_kwargs["time"]
+            elif "slot_time" in decorator_kwargs:
+                time = state.genesis_time + (decorator_kwargs["slot_time"] * spec.config.SECONDS_PER_SLOT)
+
+            head = state
+            head_epoch = spec.compute_epoch_at_slot(head.slot)
+            fork_version = spec.compute_fork_version(head_epoch)
+            fork_digest = spec.compute_fork_digest(fork_version, state.genesis_validators_root)
+            finalized_checkpoint = state.finalized_checkpoint
+
+            yield 'hive_config', {
+                'genesis_time': int(state.genesis_time),
+                'genesis_validators_root': state.genesis_validators_root.hex(),
+                'time': int(time),
+                'fork_version': fork_version.hex(),
+                'fork_digest': fork_digest.hex(),
+                'finalized_checkpoint': {
+                    'epoch': int(finalized_checkpoint.epoch),
+                    'root': finalized_checkpoint.root.hex(),
+                },
+                'head': {
+                    'epoch': int(head_epoch),
+                    'root': head.hash_tree_root().hex(),
+                },
+                'head_slot': int(head.slot),
+            }
+        return wrapper
+    return decorator

tests/core/pyspec/eth2spec/test/helpers/constants.py

@@ -50,6 +50,7 @@
 #
 MAINNET = PresetBaseName('mainnet')
 MINIMAL = PresetBaseName('minimal')
+HIVE = PresetBaseName('hive')
 
 ALL_PRESETS = (MINIMAL, MAINNET)