Introduce blob schedule

configs/mainnet.yaml

@@ -172,7 +172,6 @@ SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 VALIDATOR_CUSTODY_REQUIREMENT: 8
 BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
-MAX_BLOBS_PER_BLOCK_FULU: 12
 MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # EIP7441
@@ -190,3 +189,14 @@ VIEW_FREEZE_DEADLINE: 9
 MAX_REQUEST_INCLUSION_LIST: 16
 # 2**13 (= 8192)
 MAX_BYTES_PER_INCLUSION_LIST: 8192
+
+# Blob Scheduling
+# ---------------------------------------------------------------
+
+BLOB_SCHEDULE:
+  # Deneb
+  - EPOCH: 269568
+    MAX_BLOBS_PER_BLOCK: 6
+  # Electra
+  - EPOCH: 364032
+    MAX_BLOBS_PER_BLOCK: 9

configs/minimal.yaml

@@ -169,7 +169,6 @@ SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 VALIDATOR_CUSTODY_REQUIREMENT: 8
 BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
-MAX_BLOBS_PER_BLOCK_FULU: 12
 MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # EIP7441
@@ -187,3 +186,14 @@ VIEW_FREEZE_DEADLINE: 3
 MAX_REQUEST_INCLUSION_LIST: 16
 # 2**13 (= 8192)
 MAX_BYTES_PER_INCLUSION_LIST: 8192
+
+# Blob Scheduling
+# ---------------------------------------------------------------
+
+BLOB_SCHEDULE:
+  # Deneb
+  - EPOCH: 18446744073709551615
+    MAX_BLOBS_PER_BLOCK: 6
+  # Electra
+  - EPOCH: 18446744073709551615
+    MAX_BLOBS_PER_BLOCK: 9

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "curdleproofs==0.1.2",
   "eth-typing==5.2.1",
   "eth-utils==5.3.0",
+  "frozendict==2.4.6",
   "lru-dict==1.3.0",
   "marko==2.1.3",
   "milagro_bls_binding==1.9.0",

pysetup/helpers.py

@@ -1,5 +1,5 @@
 import re
-from typing import TypeVar, Dict
+from typing import TypeVar, Dict, Union, List
 import textwrap
 from functools import reduce
 
@@ -85,23 +85,42 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
     ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
 
     # Access global dict of config vars for runtime configurables
+    # Ignore variable between quotes and doubles quotes
     for name in spec_object.config_vars.keys():
-        functions_spec = re.sub(r"\b%s\b" % name, 'config.' + name, functions_spec)
-        ordered_class_objects_spec = re.sub(r"\b%s\b" % name, 'config.' + name, ordered_class_objects_spec)
-
-    def format_config_var(name: str, vardef: VariableDefinition) -> str:
-        if vardef.type_name is None:
+        functions_spec = re.sub(r"(?<!['\"])\b%s\b(?!['\"])" % name, "config." + name, functions_spec)
+        ordered_class_objects_spec = re.sub(r"(?<!['\"])\b%s\b(?!['\"])" % name, "config." + name, ordered_class_objects_spec)
+
+    def format_config_var(name: str, vardef) -> str:
+        if isinstance(vardef, list):
+            # A special case for list of records.
+            indent = " " * 4
+            lines = [f"{name}=("]
+            for d in vardef:
+                line = indent*2 + "frozendict({\n"
+                for k, v in d.items():
+                    line += indent * 3 + f'"{k}": {v},\n'
+                line += indent*2 + "}),"
+                lines.append(line)
+            lines.append(indent + "),")
+            return "\n".join(lines)
+        elif vardef.type_name is None:
             out = f'{name}={vardef.value},'
         else:
             out = f'{name}={vardef.type_name}({vardef.value}),'
         if vardef.comment is not None:
             out += f'  # {vardef.comment}'
         return out
 
+    def format_config_var_param(value):
+        if isinstance(value, list):
+            # A special case for list of records.
+            return "tuple[frozendict[str, Any], ...]"
+        elif isinstance(value, VariableDefinition):
+            return value.type_name if value.type_name is not None else "int"
+
     config_spec = 'class Configuration(NamedTuple):\n'
     config_spec += '    PRESET_BASE: str\n'
-    config_spec += '\n'.join(f'    {k}: {v.type_name if v.type_name is not None else "int"}'
-                             for k, v in spec_object.config_vars.items())
+    config_spec += '\n'.join(f'    {k}: {format_config_var_param(v)}' for k, v in spec_object.config_vars.items())
     config_spec += '\n\n\nconfig = Configuration(\n'
     config_spec += f'    PRESET_BASE="{preset_name}",\n'
     config_spec += '\n'.join('    ' + format_config_var(k, v) for k, v in spec_object.config_vars.items())
@@ -267,13 +286,16 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     )
 
 
-def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]:
+def parse_config_vars(conf: Dict[str, str]) -> Dict[str, Union[str, List[Dict[str, str]]]]:
     """
     Parses a dict of basic str/int/list types into a dict for insertion into the spec code.
     """
-    out: Dict[str, str] = dict()
+    out: Dict[str, Union[str, List[Dict[str, str]]]] = dict()
     for k, v in conf.items():
-        if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'):
+        if isinstance(v, list):
+            # A special case for list of records
+            out[k] = v
+        elif isinstance(v, str) and (v.startswith("0x") or k == "PRESET_BASE" or k == "CONFIG_NAME"):
             # Represent byte data with string, to avoid misinterpretation as big-endian int.
             # Everything except PRESET_BASE and CONFIG_NAME is either byte data or an integer.
             out[k] = f"'{v}'"

pysetup/spec_builders/fulu.py

@@ -10,6 +10,7 @@ class FuluSpecBuilder(BaseSpecBuilder):
     @classmethod
     def imports(cls, preset_name: str):
         return f'''
+from frozendict import frozendict
 from eth2spec.electra import {preset_name} as electra
 '''
 

setup.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import re
 import string
 import sys
 import warnings
@@ -215,6 +216,8 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
     current_name = None
     should_skip = False
+    list_of_records = None
+    list_of_records_name = None
     for child in document.children:
         if isinstance(child, BlankLine):
             continue
@@ -255,6 +258,62 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                 ssz_objects[current_name] = "\n".join(line.rstrip() for line in source.splitlines())
             else:
                 raise Exception("unrecognized python code element: " + source)
+        elif isinstance(child, Table) and list_of_records is not None:
+            list_of_records_header = None
+            for i, row in enumerate(child.children):
+                # This will start as an empty list when there is a <!-- list-of-records --> comment,
+                # which indicates that the next table is a list-of-records. After we're done parsing
+                # the table, we will reset this to None.
+                if list_of_records is not None:
+                    if i == 0:
+                        # Save the table header, this will be used for field names
+                        # Skip the last item, which is the description
+                        list_of_records_header = [
+                            # Convert the titles to SNAKE_CASE
+                            re.sub(r'\s+', '_', value.children[0].children.upper())
+                            for value in row.children[:-1]
+                        ]
+                    else:
+                        # Add the row entry to our list of records
+                        list_of_records.append({
+                            list_of_records_header[i]: value.children[0].children
+                            for i, value in enumerate(row.children[:-1])
+                        })
+
+            # Make a type map from the spec definition
+            # We'll apply this to the file config (ie mainnet.yaml)
+            type_map: dict[str,str] = {}
+            pattern = re.compile(r'^(\w+)\(.*\)$')
+            for entry in list_of_records:
+                for k, v in entry.items():
+                    m = pattern.match(v)
+                    if m:
+                        type_map[k] = m.group(1)
+
+            # Apply the types to the file config
+            list_of_records_config: list[dict[str,str]] = []
+            for entry in config[list_of_records_name]:
+                new_entry: dict[str,str] = {}
+                for k, v in entry.items():
+                    ctor = type_map.get(k)
+                    if ctor:
+                        new_entry[k] = f"{ctor}({v})"
+                    else:
+                        new_entry[k] = v
+                list_of_records_config.append(new_entry)
+
+            # For mainnet, check that the spec config & file config are the same
+            # For minimal, we expect this to be different; just use the file config
+            if preset_name == "mainnet":
+                assert list_of_records == list_of_records_config, \
+                    f"list of records mismatch: {list_of_records} vs {list_of_records_config}"
+            elif preset_name == "minimal":
+                list_of_records = list_of_records_config
+
+            # Set the config variable and reset the global variable
+            config_vars[list_of_records_name] = list_of_records
+            list_of_records = None
+
         elif isinstance(child, Table):
             for row in child.children:
                 cells = row.children
@@ -315,6 +374,14 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
         elif isinstance(child, HTMLBlock):
             if child.body.strip() == "<!-- eth2spec: skip -->":
                 should_skip = True
+            # Handle list-of-records tables
+            match = re.match(r"<!--\s*list-of-records:([a-zA-Z0-9_-]+)\s*-->", child.body.strip())
+            if match:
+                # Initialize list-of-records, in the next iteration this will indicate that the
+                # table is a list-of-records and must be parsed differently.
+                list_of_records = []
+                # Use regex to extract the desired configuration list name
+                list_of_records_name = match.group(1).upper()
 
     # Load KZG trusted setup from files
     if any('KZG_SETUP' in name for name in constant_vars):

specs/fulu/beacon-chain.md

@@ -6,7 +6,6 @@
 
 - [Introduction](#introduction)
 - [Configuration](#configuration)
-  - [Execution](#execution)
 - [Beacon chain state transition function](#beacon-chain-state-transition-function)
   - [Block processing](#block-processing)
     - [Execution payload](#execution-payload)
@@ -20,12 +19,6 @@
 
 ## Configuration
 
-### Execution
-
-| Name                       | Value        | Description                                                                                                   |
-| -------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------- |
-| `MAX_BLOBS_PER_BLOCK_FULU` | `uint64(12)` | *[New in Fulu:EIP7594]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` |
-
 ## Beacon chain state transition function
 
 ### Block processing
@@ -45,7 +38,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi
     # Verify timestamp
     assert payload.timestamp == compute_timestamp_at_slot(state, state.slot)
     # Verify commitments are under limit
-    assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_FULU  # [Modified in Fulu:EIP7594]
+    assert len(body.blob_kzg_commitments) <= get_max_blobs_per_block(get_current_epoch(state))  # [Modified in Fulu:EIP7594]
     # Verify the execution payload is valid
     versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments]
     assert execution_engine.verify_and_notify_new_payload(

specs/fulu/das-core.md

@@ -10,11 +10,13 @@
 - [Configuration](#configuration)
   - [Data size](#data-size)
   - [Custody setting](#custody-setting)
+  - [Blob schedule](#blob-schedule)
   - [Containers](#containers)
     - [`DataColumnSidecar`](#datacolumnsidecar)
     - [`MatrixEntry`](#matrixentry)
 - [Helper functions](#helper-functions)
   - [`get_custody_groups`](#get_custody_groups)
+  - [`get_max_blobs_per_block`](#get_max_blobs_per_block)
   - [`compute_columns_for_custody_group`](#compute_columns_for_custody_group)
   - [`compute_matrix`](#compute_matrix)
   - [`recover_matrix`](#recover_matrix)
@@ -67,6 +69,17 @@ The following values are (non-configurable) constants used throughout the specif
 | `NUMBER_OF_CUSTODY_GROUPS` | `128` | Number of custody groups available for nodes to custody                           |
 | `CUSTODY_REQUIREMENT`      | `4`   | Minimum number of custody groups an honest node custodies and serves samples from |
 
+### Blob schedule
+
+*[New in EIP7594]* This schedule defines the maximum blobs per block limit for a given epoch.
+
+<!-- list-of-records:blob_schedule -->
+
+| Epoch                       | Max Blobs Per Block | Description                      |
+| --------------------------- | ------------------- | -------------------------------- |
+| `Epoch(269568)` **Deneb**   | `uint64(6)`         | The limit is set to `6` blobs    |
+| `Epoch(364032)` **Electra** | `uint64(9)`         | The limit is raised to `9` blobs |
+
 ### Containers
 
 #### `DataColumnSidecar`
@@ -118,6 +131,20 @@ def get_custody_groups(node_id: NodeID, custody_group_count: uint64) -> Sequence
     return sorted(custody_groups)
 ```
 
+### `get_max_blobs_per_block`
+
+```python
+def get_max_blobs_per_block(epoch: Epoch) -> uint64:
+    """
+    Return the maximum number of blobs that can be included in a block for a given epoch.
+    """
+    assert len(BLOB_SCHEDULE) > 0
+    for entry in sorted(BLOB_SCHEDULE, key=lambda e: e["EPOCH"], reverse=True):
+        if epoch >= entry["EPOCH"]:
+            return entry["MAX_BLOBS_PER_BLOCK"]
+    return min(entry["MAX_BLOBS_PER_BLOCK"] for entry in BLOB_SCHEDULE)
+```
+
 ### `compute_columns_for_custody_group`
 
 ```python

specs/fulu/p2p-interface.md

@@ -167,7 +167,7 @@ Some gossip meshes are upgraded in the Fulu fork to support upgraded types.
 *Updated validation*
 
 - _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer --
-  i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_FULU`
+  i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= get_max_blobs_per_block(get_current_epoch(state))`
 
 ##### Blob subnets
 

tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py

@@ -411,7 +411,7 @@ def test_invalid_exceed_max_blobs_per_block(spec, state):
     execution_payload = build_empty_execution_payload(spec, state)
 
     opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(
-        spec, blob_count=get_max_blob_count(spec) + 1
+        spec, blob_count=get_max_blob_count(spec, state) + 1
     )
 
     execution_payload.transactions = [opaque_tx]

tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py

@@ -84,36 +84,38 @@ def test_one_blob_two_txs(spec, state):
 @with_deneb_until_eip7732
 @spec_state_test
 def test_one_blob_max_txs(spec, state):
-    yield from run_block_with_blobs(spec, state, blob_count=1, tx_count=get_max_blob_count(spec))
+    yield from run_block_with_blobs(
+        spec, state, blob_count=1, tx_count=get_max_blob_count(spec, state)
+    )
 
 
 @with_deneb_until_eip7732
 @spec_state_test
 def test_invalid_one_blob_max_plus_one_txs(spec, state):
     yield from run_block_with_blobs(
-        spec, state, blob_count=1, tx_count=get_max_blob_count(spec) + 1, valid=False
+        spec, state, blob_count=1, tx_count=get_max_blob_count(spec, state) + 1, valid=False
     )
 
 
 @with_deneb_until_eip7732
 @spec_state_test
 def test_max_blobs_per_block(spec, state):
-    yield from run_block_with_blobs(spec, state, blob_count=get_max_blob_count(spec))
+    yield from run_block_with_blobs(spec, state, blob_count=get_max_blob_count(spec, state))
 
 
 @with_deneb_until_eip7732
 @spec_state_test
 def test_invalid_max_blobs_per_block_two_txs(spec, state):
     yield from run_block_with_blobs(
-        spec, state, blob_count=get_max_blob_count(spec), tx_count=2, valid=False
+        spec, state, blob_count=get_max_blob_count(spec, state), tx_count=2, valid=False
     )
 
 
 @with_deneb_until_eip7732
 @spec_state_test
 def test_invalid_exceed_max_blobs_per_block(spec, state):
     yield from run_block_with_blobs(
-        spec, state, blob_count=get_max_blob_count(spec) + 1, valid=False
+        spec, state, blob_count=get_max_blob_count(spec, state) + 1, valid=False
     )
 
 

tests/core/pyspec/eth2spec/test/fulu/merkle_proof/test_single_merkle_proof.py

@@ -82,7 +82,7 @@ def test_blob_kzg_commitments_merkle_proof__random_block_1(spec, state):
 @with_fulu_and_later
 @spec_state_test
 def test_blob_kzg_commitments_merkle_proof__multiple_blobs(spec, state):
-    blob_count = spec.config.MAX_BLOBS_PER_BLOCK_FULU // 2
+    blob_count = spec.get_max_blobs_per_block(spec.get_current_epoch(state)) // 2
     rng = random.Random(2222)
     yield from _run_blob_kzg_commitments_merkle_proof_test(
         spec, state, rng=rng, blob_count=blob_count
@@ -93,7 +93,7 @@ def test_blob_kzg_commitments_merkle_proof__multiple_blobs(spec, state):
 @with_fulu_and_later
 @spec_state_test
 def test_blob_kzg_commitments_merkle_proof__max_blobs(spec, state):
-    max_blobs = spec.config.MAX_BLOBS_PER_BLOCK_FULU
+    max_blobs = spec.get_max_blobs_per_block(spec.get_current_epoch(state))
     rng = random.Random(3333)
     yield from _run_blob_kzg_commitments_merkle_proof_test(
         spec, state, rng=rng, blob_count=max_blobs