Make CUSTODY_REQUIREMENT unit be subnets; move some depended helper…

…s to `das-core.md`

configs/mainnet.yaml

@@ -157,3 +157,6 @@ BLOB_SIDECAR_SUBNET_COUNT: 6
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256
 # `Epoch(2)`
 WHISK_PROPOSER_SELECTION_GAP: 2
+
+# EIP7594
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32

configs/minimal.yaml

@@ -62,7 +62,6 @@ WHISK_FORK_EPOCH: 18446744073709551615
 EIP7594_FORK_VERSION: 0x06000001
 EIP7594_FORK_EPOCH: 18446744073709551615
 
-
 # Time parameters
 # ---------------------------------------------------------------
 # [customized] Faster for testing purposes
@@ -156,3 +155,6 @@ BLOB_SIDECAR_SUBNET_COUNT: 6
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4
 WHISK_PROPOSER_SELECTION_GAP: 1
+
+# EIP7594
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32

specs/_features/eip7594/das-core.md

@@ -13,7 +13,7 @@
   - [Data size](#data-size)
   - [Custody setting](#custody-setting)
   - [Helper functions](#helper-functions)
-    - [`get_custody_lines`](#get_custody_lines)
+    - [`get_custody_columns`](#get_custody_columns)
     - [`compute_extended_data`](#compute_extended_data)
     - [`compute_extended_matrix`](#compute_extended_matrix)
     - [`get_data_column_sidecars`](#get_data_column_sidecars)
@@ -54,23 +54,51 @@ We define the following Python custom types for type hinting and readability:
 | - | - | - |
 | `NUMBER_OF_COLUMNS` | `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) | Number of columns in the extended data matrix. |
 
+### Networking
+
+| Name                               | Value            | Description                                                         |
+|------------------------------------|------------------|---------------------------------------------------------------------|
+| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32`             | The number of data column sidecar subnets used in the gossipsub protocol.  |
+
 ### Custody setting
 
 | Name | Value | Description |
 | - | - | - |
 | `SAMPLES_PER_SLOT` | `8` | Number of `DataColumn` random samples a node queries per slot |
-| `CUSTODY_REQUIREMENT` | `2` | Minimum number of columns an honest node custodies and serves samples from |
+| `CUSTODY_REQUIREMENT` | `1` | Minimum number of subnets an honest node custodies and serves samples from |
 | `TARGET_NUMBER_OF_PEERS` | `70` | Suggested minimum peer count |
 
+### Containers
+
+#### `DataColumnSidecar`
+
+```python
+class DataColumnSidecar(Container):
+    index: ColumnIndex  # Index of column in extended matrix
+    column: DataColumn
+    kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK]
+    kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK]
+    signed_block_header: SignedBeaconBlockHeader
+    kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH]
+```
+
 ### Helper functions
 
-#### `get_custody_lines`
+#### `get_custody_columns`
 
 ```python
-def get_custody_lines(node_id: NodeID, custody_size: uint64) -> Sequence[ColumnIndex]:
-    assert custody_size <= NUMBER_OF_COLUMNS
-    column_index = node_id % NUMBER_OF_COLUMNS
-    return [ColumnIndex((column_index + i) % NUMBER_OF_COLUMNS) for i in range(custody_size)]
+def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]:
+    assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT
+    subnet_ids = [
+        bytes_to_uint64(hash(uint_to_bytes(uint64(node_id + i)))[0:8]) % DATA_COLUMN_SIDECAR_SUBNET_COUNT
+        for i in range(custody_subnet_count)
+    ]
+    columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT
+    return [
+        ColumnIndex(subnet_id + (i * columns_per_subnet))
+        for i in range(columns_per_subnet)
+        for subnet_id in subnet_ids
+    ]
 ```
 
 #### `compute_extended_data`
@@ -126,15 +154,15 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock,
 
 ### Custody requirement
 
-Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` columns per slot. The particular columns that the node is required to custody are selected pseudo-randomly (more on this below).
+Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. The particular columns that the node is required to custody are selected pseudo-randomly (more on this below).
 
-A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT`  via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 8` if the node custodies `8` columns each slot) -- up to a `NUMBER_OF_COLUMNS` (i.e. a super-full node).
+A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node).
 
 A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns.
 
 ### Public, deterministic selection
 
-The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_lines`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public.
+The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public.
 
 *Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody.
 
@@ -178,7 +206,7 @@ Once the node obtain the column, the node should send the missing columns to the
 
 ## Peer sampling
 
-At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarByRoot` request. A node utilizes `get_custody_lines` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success.
+At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success.
 
 ## Peer scoring
 
@@ -220,4 +248,4 @@ However, for simplicity, we don't assign row custody assignments to nodes in the
 
 ### Subnet stability
 
-To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_lines`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs.
+To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs.

specs/_features/eip7594/p2p-interface.md

@@ -37,26 +37,8 @@
 |------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
 | `KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH`   | `uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')))` (= 4) | <!-- predefined --> Merkle proof index for `blob_kzg_commitments` |
 
-### Configuration
-
-| Name                                     | Value                             | Description                                                         |
-|------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
-| `DATA_COLUMN_SIDECAR_SUBNET_COUNT`              | `32`                               | The number of data column sidecar subnets used in the gossipsub protocol.  |
-
 ### Containers
 
-#### `DataColumnSidecar`
-
-```python
-class DataColumnSidecar(Container):
-    index: ColumnIndex  # Index of column in extended matrix
-    column: DataColumn
-    kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK]
-    kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK]
-    signed_block_header: SignedBeaconBlockHeader
-    kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH]
-```
-
 #### `DataColumnIdentifier`
 
 ```python
@@ -111,7 +93,6 @@ def compute_subnet_for_data_column_sidecar(column_index: ColumnIndex) -> SubnetI
     return SubnetID(column_index % DATA_COLUMN_SIDECAR_SUBNET_COUNT)
 ```
 
-
 ### The gossip domain: gossipsub
 
 Some gossip meshes are upgraded in the EIP-7594 fork to support upgraded types.

tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py

@@ -12,6 +12,6 @@ def test_invariants(spec):
     assert spec.FIELD_ELEMENTS_PER_BLOB % spec.FIELD_ELEMENTS_PER_CELL == 0
     assert spec.FIELD_ELEMENTS_PER_BLOB * 2 % spec.NUMBER_OF_COLUMNS == 0
     assert spec.SAMPLES_PER_SLOT <= spec.NUMBER_OF_COLUMNS
-    assert spec.CUSTODY_REQUIREMENT <= spec.NUMBER_OF_COLUMNS
-    assert spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT <= spec.NUMBER_OF_COLUMNS
-    assert spec.NUMBER_OF_COLUMNS % spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0
+    assert spec.CUSTODY_REQUIREMENT <= spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT
+    assert spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT <= spec.NUMBER_OF_COLUMNS
+    assert spec.NUMBER_OF_COLUMNS % spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0

tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py

@@ -6,36 +6,47 @@
 )
 
 
+def run_get_custody_columns(spec, peer_count, custody_subnet_count):
+    assignments = [spec.get_custody_columns(node_id, custody_subnet_count) for node_id in range(peer_count)]
+
+    subnet_per_column = spec.NUMBER_OF_COLUMNS // spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT
+    for assignment in assignments:
+        assert len(assignment) == custody_subnet_count * subnet_per_column
+
+
 @with_eip7594_and_later
 @spec_test
 @single_phase
-def test_get_custody_lines_peers_within_number_of_columns(spec):
+def test_get_custody_columns_peers_within_number_of_columns(spec):
     peer_count = 10
-    custody_size = spec.CUSTODY_REQUIREMENT
+    custody_subnet_count = spec.CUSTODY_REQUIREMENT
     assert spec.NUMBER_OF_COLUMNS > peer_count
-    assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)]
-
-    for assignment in assignments:
-        assert len(assignment) == custody_size
+    run_get_custody_columns(spec, peer_count, custody_subnet_count)
 
 
 @with_eip7594_and_later
 @spec_test
 @single_phase
-def test_get_custody_lines_peers_more_than_number_of_columns(spec):
+def test_get_custody_columns_peers_more_than_number_of_columns(spec):
     peer_count = 200
-    custody_size = spec.CUSTODY_REQUIREMENT
+    custody_subnet_count = spec.CUSTODY_REQUIREMENT
     assert spec.NUMBER_OF_COLUMNS < peer_count
-    assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)]
+    run_get_custody_columns(spec, peer_count, custody_subnet_count)
 
-    for assignment in assignments:
-        assert len(assignment) == custody_size
+
+@with_eip7594_and_later
+@spec_test
+@single_phase
+def test_get_custody_columns_maximum_subnets(spec):
+    peer_count = 10
+    custody_subnet_count = spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT
+    run_get_custody_columns(spec, peer_count, custody_subnet_count)
 
 
 @with_eip7594_and_later
 @spec_test
 @single_phase
-def test_get_custody_lines_custody_size_more_than_number_of_columns(spec):
+def test_get_custody_columns_custody_size_more_than_number_of_columns(spec):
     node_id = 1
-    custody_size = spec.NUMBER_OF_COLUMNS + 1
-    expect_assertion_error(lambda: spec.get_custody_lines(node_id, custody_size))
+    custody_subnet_count = spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + 1
+    expect_assertion_error(lambda: spec.get_custody_columns(node_id, custody_subnet_count))

tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py

@@ -10,10 +10,10 @@
 @single_phase
 def test_compute_subnet_for_data_column_sidecar(spec):
     subnet_results = []
-    for column_index in range(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT):
+    for column_index in range(spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT):
         subnet_results.append(spec.compute_subnet_for_data_column_sidecar(column_index))
     # no duplicates
     assert len(subnet_results) == len(set(subnet_results))
     # next one should be duplicate
-    next_subnet = spec.compute_subnet_for_data_column_sidecar(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT)
+    next_subnet = spec.compute_subnet_for_data_column_sidecar(spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT)
     assert next_subnet == subnet_results[0]