Debug data columns API endpoint (#15701)

* initial

* fixing from self review

* changelog

* fixing endpoints and adding test

* removed unneeded test

* self review

* fixing mock columns

* fixing tests

* gofmt

* fixing endpoint

* gaz

* gofmt

* fixing tests

* gofmt

* gaz

* radek comments

* gaz

* fixing formatting

* deduplicating and fixing an old bug, will break into separate PR

* better way for version

* optimizing post merge and fixing tests

* Update beacon-chain/rpc/eth/debug/handlers.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update beacon-chain/rpc/eth/debug/handlers.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <[email protected]>

* adding some of radek's feedback

* reverting and gaz

---------

Co-authored-by: Radosław Kapka <[email protected]>

Author: james-prysm

api/server/structs/endpoints_debug.go

@@ -56,3 +56,19 @@ type ForkChoiceNodeExtraData struct {
 	TimeStamp                string `json:"timestamp"`
 	Target                   string `json:"target"`
 }
+
+type GetDebugDataColumnSidecarsResponse struct {
+	Version             string               `json:"version"`
+	ExecutionOptimistic bool                 `json:"execution_optimistic"`
+	Finalized           bool                 `json:"finalized"`
+	Data                []*DataColumnSidecar `json:"data"`
+}
+
+type DataColumnSidecar struct {
+	Index                        string                   `json:"index"`
+	Column                       []string                 `json:"column"`
+	KzgCommitments               []string                 `json:"kzg_commitments"`
+	KzgProofs                    []string                 `json:"kzg_proofs"`
+	SignedBeaconBlockHeader      *SignedBeaconBlockHeader `json:"signed_block_header"`
+	KzgCommitmentsInclusionProof []string                 `json:"kzg_commitments_inclusion_proof"`
+}

beacon-chain/rpc/endpoints.go

@@ -106,7 +106,7 @@ func (s *Service) endpoints(
 	}
 
 	if enableDebug {
-		endpoints = append(endpoints, s.debugEndpoints(stater)...)
+		endpoints = append(endpoints, s.debugEndpoints(stater, blocker)...)
 	}
 
 	return endpoints
@@ -1097,7 +1097,7 @@ func (s *Service) lightClientEndpoints() []endpoint {
 	}
 }
 
-func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
+func (s *Service) debugEndpoints(stater lookup.Stater, blocker lookup.Blocker) []endpoint {
 	server := &debug.Server{
 		BeaconDB:              s.cfg.BeaconDB,
 		HeadFetcher:           s.cfg.HeadFetcher,
@@ -1107,6 +1107,8 @@ func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
 		ForkchoiceFetcher:     s.cfg.ForkchoiceFetcher,
 		FinalizationFetcher:   s.cfg.FinalizationFetcher,
 		ChainInfoFetcher:      s.cfg.ChainInfoFetcher,
+		GenesisTimeFetcher:    s.cfg.GenesisTimeFetcher,
+		Blocker:               blocker,
 	}
 
 	const namespace = "debug"
@@ -1141,6 +1143,16 @@ func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
 			handler: server.GetForkChoice,
 			methods: []string{http.MethodGet},
 		},
+		{
+			template: "/eth/v1/debug/beacon/data_column_sidecars/{block_id}",
+			name:     namespace + ".GetDataColumnSidecars",
+			middleware: []middleware.Middleware{
+				middleware.AcceptHeaderHandler([]string{api.JsonMediaType, api.OctetStreamMediaType}),
+				middleware.AcceptEncodingHeaderHandler(),
+			},
+			handler: server.DataColumnSidecars,
+			methods: []string{http.MethodGet},
+		},
 	}
 }
 

beacon-chain/rpc/endpoints_test.go

@@ -80,9 +80,10 @@ func Test_endpoints(t *testing.T) {
 	}
 
 	debugRoutes := map[string][]string{
-		"/eth/v2/debug/beacon/states/{state_id}": {http.MethodGet},
-		"/eth/v2/debug/beacon/heads":             {http.MethodGet},
-		"/eth/v1/debug/fork_choice":              {http.MethodGet},
+		"/eth/v2/debug/beacon/states/{state_id}":               {http.MethodGet},
+		"/eth/v2/debug/beacon/heads":                           {http.MethodGet},
+		"/eth/v1/debug/fork_choice":                            {http.MethodGet},
+		"/eth/v1/debug/beacon/data_column_sidecars/{block_id}": {http.MethodGet},
 	}
 
 	eventsRoutes := map[string][]string{

beacon-chain/rpc/eth/debug/BUILD.bazel

@@ -13,13 +13,19 @@ go_library(
         "//api/server/structs:go_default_library",
         "//beacon-chain/blockchain:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/rpc/core:go_default_library",
         "//beacon-chain/rpc/eth/helpers:go_default_library",
         "//beacon-chain/rpc/eth/shared:go_default_library",
         "//beacon-chain/rpc/lookup:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//monitoring/tracing/trace:go_default_library",
         "//network/httputil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
     ],
 )
 
@@ -34,9 +40,13 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/forkchoice/types:go_default_library",
+        "//beacon-chain/rpc/core:go_default_library",
         "//beacon-chain/rpc/testutil:go_default_library",
         "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",

beacon-chain/rpc/eth/debug/handlers.go

@@ -4,19 +4,31 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
 
 	"github.com/OffchainLabs/prysm/v6/api"
 	"github.com/OffchainLabs/prysm/v6/api/server/structs"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/helpers"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/shared"
+	"github.com/OffchainLabs/prysm/v6/config/params"
+	"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
+	"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
 	"github.com/OffchainLabs/prysm/v6/monitoring/tracing/trace"
 	"github.com/OffchainLabs/prysm/v6/network/httputil"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
 	"github.com/OffchainLabs/prysm/v6/runtime/version"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
 )
 
-const errMsgStateFromConsensus = "Could not convert consensus state to response"
+const (
+	errMsgStateFromConsensus = "Could not convert consensus state to response"
+)
 
 // GetBeaconStateV2 returns the full beacon state for a given state ID.
 func (s *Server) GetBeaconStateV2(w http.ResponseWriter, r *http.Request) {
@@ -208,3 +220,181 @@ func (s *Server) GetForkChoice(w http.ResponseWriter, r *http.Request) {
 	}
 	httputil.WriteJson(w, resp)
 }
+
+// DataColumnSidecars retrieves data column sidecars for a given block id.
+func (s *Server) DataColumnSidecars(w http.ResponseWriter, r *http.Request) {
+	ctx, span := trace.StartSpan(r.Context(), "debug.DataColumnSidecars")
+	defer span.End()
+
+	// Check if we're before Fulu fork - data columns are only available from Fulu onwards
+	fuluForkEpoch := params.BeaconConfig().FuluForkEpoch
+	if fuluForkEpoch == math.MaxUint64 {
+		httputil.HandleError(w, "Data columns are not supported - Fulu fork not configured", http.StatusBadRequest)
+		return
+	}
+
+	// Check if we're before Fulu fork based on current slot
+	currentSlot := s.GenesisTimeFetcher.CurrentSlot()
+	currentEpoch := primitives.Epoch(currentSlot / params.BeaconConfig().SlotsPerEpoch)
+	if currentEpoch < fuluForkEpoch {
+		httputil.HandleError(w, "Data columns are not supported - before Fulu fork", http.StatusBadRequest)
+		return
+	}
+
+	indices, err := parseDataColumnIndices(r.URL)
+	if err != nil {
+		httputil.HandleError(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	segments := strings.Split(r.URL.Path, "/")
+	blockId := segments[len(segments)-1]
+
+	verifiedDataColumns, rpcErr := s.Blocker.DataColumns(ctx, blockId, indices)
+	if rpcErr != nil {
+		code := core.ErrorReasonToHTTP(rpcErr.Reason)
+		switch code {
+		case http.StatusBadRequest:
+			httputil.HandleError(w, "Bad request: "+rpcErr.Err.Error(), code)
+			return
+		case http.StatusNotFound:
+			httputil.HandleError(w, "Not found: "+rpcErr.Err.Error(), code)
+			return
+		case http.StatusInternalServerError:
+			httputil.HandleError(w, "Internal server error: "+rpcErr.Err.Error(), code)
+			return
+		default:
+			httputil.HandleError(w, rpcErr.Err.Error(), code)
+			return
+		}
+	}
+
+	blk, err := s.Blocker.Block(ctx, []byte(blockId))
+	if err != nil {
+		httputil.HandleError(w, "Could not fetch block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if blk == nil {
+		httputil.HandleError(w, "Block not found", http.StatusNotFound)
+		return
+	}
+
+	if httputil.RespondWithSsz(r) {
+		sszResp, err := buildDataColumnSidecarsSSZResponse(verifiedDataColumns)
+		if err != nil {
+			httputil.HandleError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set(api.VersionHeader, version.String(blk.Version()))
+		httputil.WriteSsz(w, sszResp)
+		return
+	}
+
+	blkRoot, err := blk.Block().HashTreeRoot()
+	if err != nil {
+		httputil.HandleError(w, "Could not hash block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	isOptimistic, err := s.OptimisticModeFetcher.IsOptimisticForRoot(ctx, blkRoot)
+	if err != nil {
+		httputil.HandleError(w, "Could not check if block is optimistic: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	data := buildDataColumnSidecarsJsonResponse(verifiedDataColumns)
+	resp := &structs.GetDebugDataColumnSidecarsResponse{
+		Version:             version.String(blk.Version()),
+		Data:                data,
+		ExecutionOptimistic: isOptimistic,
+		Finalized:           s.FinalizationFetcher.IsFinalized(ctx, blkRoot),
+	}
+	w.Header().Set(api.VersionHeader, version.String(blk.Version()))
+	httputil.WriteJson(w, resp)
+}
+
+// parseDataColumnIndices filters out invalid and duplicate data column indices
+func parseDataColumnIndices(url *url.URL) ([]int, error) {
+	numberOfColumns := params.BeaconConfig().NumberOfColumns
+	rawIndices := url.Query()["indices"]
+	indices := make([]int, 0, numberOfColumns)
+	invalidIndices := make([]string, 0)
+loop:
+	for _, raw := range rawIndices {
+		ix, err := strconv.Atoi(raw)
+		if err != nil {
+			invalidIndices = append(invalidIndices, raw)
+			continue
+		}
+		if !(0 <= ix && uint64(ix) < numberOfColumns) {
+			invalidIndices = append(invalidIndices, raw)
+			continue
+		}
+		for i := range indices {
+			if ix == indices[i] {
+				continue loop
+			}
+		}
+		indices = append(indices, ix)
+	}
+
+	if len(invalidIndices) > 0 {
+		return nil, fmt.Errorf("requested data column indices %v are invalid", invalidIndices)
+	}
+	return indices, nil
+}
+
+func buildDataColumnSidecarsJsonResponse(verifiedDataColumns []blocks.VerifiedRODataColumn) []*structs.DataColumnSidecar {
+	sidecars := make([]*structs.DataColumnSidecar, len(verifiedDataColumns))
+	for i, dc := range verifiedDataColumns {
+		column := make([]string, len(dc.Column))
+		for j, cell := range dc.Column {
+			column[j] = hexutil.Encode(cell)
+		}
+
+		kzgCommitments := make([]string, len(dc.KzgCommitments))
+		for j, commitment := range dc.KzgCommitments {
+			kzgCommitments[j] = hexutil.Encode(commitment)
+		}
+
+		kzgProofs := make([]string, len(dc.KzgProofs))
+		for j, proof := range dc.KzgProofs {
+			kzgProofs[j] = hexutil.Encode(proof)
+		}
+
+		kzgCommitmentsInclusionProof := make([]string, len(dc.KzgCommitmentsInclusionProof))
+		for j, proof := range dc.KzgCommitmentsInclusionProof {
+			kzgCommitmentsInclusionProof[j] = hexutil.Encode(proof)
+		}
+
+		sidecars[i] = &structs.DataColumnSidecar{
+			Index:                        strconv.FormatUint(dc.Index, 10),
+			Column:                       column,
+			KzgCommitments:               kzgCommitments,
+			KzgProofs:                    kzgProofs,
+			SignedBeaconBlockHeader:      structs.SignedBeaconBlockHeaderFromConsensus(dc.SignedBlockHeader),
+			KzgCommitmentsInclusionProof: kzgCommitmentsInclusionProof,
+		}
+	}
+	return sidecars
+}
+
+// buildDataColumnSidecarsSSZResponse builds SSZ response for data column sidecars
+func buildDataColumnSidecarsSSZResponse(verifiedDataColumns []blocks.VerifiedRODataColumn) ([]byte, error) {
+	if len(verifiedDataColumns) == 0 {
+		return []byte{}, nil
+	}
+
+	// Pre-allocate buffer for all sidecars using the known SSZ size
+	sizePerSidecar := (&ethpb.DataColumnSidecar{}).SizeSSZ()
+	ssz := make([]byte, 0, sizePerSidecar*len(verifiedDataColumns))
+
+	// Marshal and append each sidecar
+	for i, sidecar := range verifiedDataColumns {
+		sszrep, err := sidecar.MarshalSSZ()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to marshal data column sidecar at index %d", i)
+		}
+		ssz = append(ssz, sszrep...)
+	}
+
+	return ssz, nil
+}