Update consensus spec to v1.6.0-alpha.4 and implement data column support

WORKSPACE

@@ -253,16 +253,16 @@ filegroup(
     url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz",
 )
 
-consensus_spec_version = "v1.6.0-alpha.1"
+consensus_spec_version = "v1.6.0-alpha.4"
 
 load("@prysm//tools:download_spectests.bzl", "consensus_spec_tests")
 
 consensus_spec_tests(
     name = "consensus_spec_tests",
     flavors = {
-        "general": "sha256-o4t9p3R+fQHF4KOykGmwlG3zDw5wUdVWprkzId8aIsk=",
-        "minimal": "sha256-sU7ToI8t3MR8x0vVjC8ERmAHZDWpEmnAC9FWIpHi5x4=",
-        "mainnet": "sha256-YKS4wngg0LgI9Upp4MYJ77aG+8+e/G4YeqEIlp06LZw=",
+        "general": "sha256-MaN4zu3o0vWZypUHS5r4D8WzJF4wANoadM8qm6iyDs4=",
+        "minimal": "sha256-aZGNPp/bBvJgq3Wf6vyR0H6G3DOkbSuggEmOL4jEmtg=",
+        "mainnet": "sha256-C7jjosvpzUgw3GPajlsWBV02ZbkZ5Uv4ikmOqfDGajI=",
     },
     version = consensus_spec_version,
 )
@@ -278,7 +278,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    integrity = "sha256-Nv4TEuEJPQIM4E6T9J0FOITsmappmXZjGtlhe1HEXnU=",
+    integrity = "sha256-qreawRS77l8CebiNww8z727qUItw7KlHY1Xqj7IrPdk=",
     strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
     url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
 )

beacon-chain/verification/data_column.go

@@ -47,6 +47,10 @@ var (
 		RequireSidecarKzgProofVerified,
 	}
 
+	// SpectestDataColumnSidecarRequirements is used by the forkchoice spectests when verifying data columns used in the on_block tests.
+	SpectestDataColumnSidecarRequirements = requirementList(GossipDataColumnSidecarRequirements).excluding(
+		RequireSidecarParentSeen, RequireSidecarParentValid)
+
 	errColumnsInvalid = errors.New("data columns failed verification")
 	errBadTopicLength = errors.New("topic length is invalid")
 	errBadTopic       = errors.New("topic is not of the one expected")

changelog/ttsao_update-consensus-spec-v160-alpha4.md

@@ -0,0 +1,3 @@
+### Changed
+
+- Update consensus spec to v1.6.0-alpha.4 and implement data column support for forkchoice spectests

config/params/loader_test.go

@@ -37,11 +37,12 @@ var placeholderFields = []string{
 	"EIP7805_FORK_EPOCH",
 	"EIP7805_FORK_VERSION",
 	"EPOCHS_PER_SHUFFLING_PHASE",
+	"INCLUSION_LIST_SUBMISSION_DEADLINE",
 	"MAX_BYTES_PER_INCLUSION_LIST",
 	"MAX_REQUEST_BLOB_SIDECARS_FULU",
 	"MAX_REQUEST_INCLUSION_LIST",
 	"MAX_REQUEST_PAYLOADS", // Compile time constant on BeaconBlockBody.ExecutionRequests
-	"PROPOSER_INCLUSION_LIST_CUT_OFF",
+	"PROPOSER_INCLUSION_LIST_CUTOFF",
 	"PROPOSER_SCORE_BOOST_EIP7732",
 	"PROPOSER_SELECTION_GAP",
 	"TARGET_NUMBER_OF_PEERS",

testing/spectest/shared/common/forkchoice/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositsnapshot:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db/filesystem:go_default_library",

testing/spectest/shared/common/forkchoice/runner.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/helpers"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/transition"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/state"
 	state_native "github.com/OffchainLabs/prysm/v6/beacon-chain/state/state-native"
@@ -62,6 +63,7 @@ func runTest(t *testing.T, config string, fork int, basePath string) { // nolint
 
 		for _, folder := range testFolders {
 			t.Run(folder.Name(), func(t *testing.T) {
+				helpers.ClearCache()
 				preStepsFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "steps.yaml")
 				require.NoError(t, err)
 				var steps []Step
@@ -148,6 +150,9 @@ func runTest(t *testing.T, config string, fork int, basePath string) { // nolint
 						}
 					}
 					runBlobStep(t, step, beaconBlock, fork, folder, testsFolderPath, builder)
+					if len(step.DataColumns) > 0 {
+						runDataColumnStep(t, step, beaconBlock, fork, folder, testsFolderPath, builder)
+					}
 					if beaconBlock != nil {
 						if step.Valid != nil && !*step.Valid {
 							builder.InvalidBlock(t, beaconBlock)
@@ -293,10 +298,154 @@ func runBlobStep(t *testing.T,
 	}
 }
 
+func runDataColumnStep(t *testing.T,
+	step Step,
+	beaconBlock interfaces.ReadOnlySignedBeaconBlock,
+	fork int,
+	folder os.DirEntry,
+	testsFolderPath string,
+	builder *Builder,
+) {
+	columnFiles := step.DataColumns
+
+	require.NotNil(t, beaconBlock)
+	require.Equal(t, true, fork >= version.Fulu)
+
+	block := beaconBlock.Block()
+	root, err := block.HashTreeRoot()
+	require.NoError(t, err)
+	kzgs, err := block.Body().BlobKzgCommitments()
+	require.NoError(t, err)
+	sh, err := beaconBlock.Header()
+	require.NoError(t, err)
+	// Use the same error that the verification system returns for data columns
+	errDataColumnsInvalid := errors.New("data columns failed verification")
+	requireVerifyExpected := errAssertionForStep(step, errDataColumnsInvalid)
+
+	var allColumns []blocks.RODataColumn
+
+	for columnIndex, columnFile := range columnFiles {
+		if columnFile == nil || *columnFile == "null" {
+			continue
+		}
+
+		dataColumnFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), fmt.Sprint(*columnFile, ".ssz_snappy"))
+		require.NoError(t, err)
+		dataColumnSSZ, err := snappy.Decode(nil /* dst */, dataColumnFile)
+		require.NoError(t, err)
+
+		var pb *ethpb.DataColumnSidecar
+
+		if step.Valid != nil && !*step.Valid {
+			pb = &ethpb.DataColumnSidecar{}
+			if err := pb.UnmarshalSSZ(dataColumnSSZ); err != nil {
+				pb = &ethpb.DataColumnSidecar{
+					Index:             uint64(columnIndex),
+					Column:            [][]byte{},
+					KzgCommitments:    kzgs,
+					KzgProofs:         make([][]byte, 0),
+					SignedBlockHeader: sh,
+				}
+			}
+		} else {
+			numCells := len(kzgs)
+			column := make([][]byte, numCells)
+			for cellIndex := 0; cellIndex < numCells; cellIndex++ {
+				cell := make([]byte, 2048)
+				cellStart := cellIndex * 2048
+				cellEnd := cellStart + 2048
+				if cellEnd <= len(dataColumnSSZ) {
+					copy(cell, dataColumnSSZ[cellStart:cellEnd])
+				}
+				column[cellIndex] = cell
+			}
+
+			inclusionProof, err := blocks.MerkleProofKZGCommitments(block.Body())
+			require.NoError(t, err)
+
+			pb = &ethpb.DataColumnSidecar{
+				Index:                        uint64(columnIndex),
+				Column:                       column,
+				KzgCommitments:               kzgs,
+				SignedBlockHeader:            sh,
+				KzgCommitmentsInclusionProof: inclusionProof,
+			}
+		}
+
+		ro, err := blocks.NewRODataColumnWithRoot(pb, root)
+		require.NoError(t, err)
+		allColumns = append(allColumns, ro)
+	}
+
+	if len(allColumns) > 0 {
+		ini, err := builder.vwait.WaitForInitializer(context.Background())
+		require.NoError(t, err)
+		// Use different verification requirements based on whether this is a valid or invalid test case
+		var forkchoiceReqs []verification.Requirement
+		if step.Valid != nil && !*step.Valid {
+			forkchoiceReqs = verification.SpectestDataColumnSidecarRequirements
+		} else {
+			forkchoiceReqs = []verification.Requirement{
+				verification.RequireNotFromFutureSlot,
+				verification.RequireSlotAboveFinalized,
+				verification.RequireValidProposerSignature,
+				verification.RequireSidecarParentSlotLower,
+				verification.RequireSidecarDescendsFromFinalized,
+				verification.RequireSidecarInclusionProven,
+				verification.RequireSidecarProposerExpected,
+			}
+		}
+		dv := ini.NewDataColumnsVerifier(allColumns, forkchoiceReqs)
+		ctx := t.Context()
+
+		if step.Valid != nil && !*step.Valid {
+			if err := dv.ValidFields(); err != nil {
+				t.Logf("ValidFields error: %s", err.Error())
+			}
+		}
+
+		if err := dv.NotFromFutureSlot(); err != nil {
+			t.Logf("NotFromFutureSlot error: %s", err.Error())
+		}
+		if err := dv.SlotAboveFinalized(); err != nil {
+			t.Logf("SlotAboveFinalized error: %s", err.Error())
+		}
+		if err := dv.SidecarInclusionProven(); err != nil {
+			t.Logf("SidecarInclusionProven error: %s", err.Error())
+		}
+		if err := dv.ValidProposerSignature(ctx); err != nil {
+			t.Logf("ValidProposerSignature error: %s", err.Error())
+		}
+		if err := dv.SidecarParentSlotLower(); err != nil {
+			t.Logf("SidecarParentSlotLower error: %s", err.Error())
+		}
+		if err := dv.SidecarDescendsFromFinalized(); err != nil {
+			t.Logf("SidecarDescendsFromFinalized error: %s", err.Error())
+		}
+		if err := dv.SidecarProposerExpected(ctx); err != nil {
+			t.Logf("SidecarProposerExpected error: %s", err.Error())
+		}
+
+		vdc, err := dv.VerifiedRODataColumns()
+		requireVerifyExpected(t, err)
+
+		if err == nil {
+			for _, column := range vdc {
+				require.NoError(t, builder.service.ReceiveDataColumn(column))
+			}
+		}
+	}
+}
+
 func errAssertionForStep(step Step, expect error) func(t *testing.T, err error) {
 	if !*step.Valid {
 		return func(t *testing.T, err error) {
-			require.ErrorIs(t, err, expect)
+			if expect.Error() == "data columns failed verification" {
+				require.NotNil(t, err)
+				require.Equal(t, true, strings.Contains(err.Error(), expect.Error()))
+			} else {
+				require.ErrorIs(t, err, expect)
+			}
 		}
 	}
 	return func(t *testing.T, err error) {

testing/spectest/shared/common/forkchoice/type.go

@@ -11,6 +11,7 @@ type Step struct {
 	PayloadStatus    *MockEngineResp `json:"payload_status"`
 	PowBlock         *string         `json:"pow_block"`
 	Check            *Check          `json:"checks"`
+	DataColumns      []*string       `json:"columns"`
 }
 
 type Check struct {