DAS storage changes

.gitignore

@@ -4,10 +4,9 @@
 *~.nib
 *.iml
 *.launch
-*.swp
 *.log
 .lh/*
-db/
+/db/
 db.version
 .classpath
 .DS_Store

data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/getBlockHeader.json

@@ -1,14 +1,14 @@
 {
   "data": {
-    "root": "0x127698f65207baf45ec6497931e82df707176d84588b76f6f1e4c4c577de3b94",
+    "root": "0x9042b435b163854b76508411d10bb170123cbe59563b862fac9d25d7493c11a9",
     "canonical": true,
     "header": {
       "message": {
         "slot": "1",
-        "proposer_index": "4666673844721362956",
-        "parent_root": "0x367cbd40ac7318427aadb97345a91fa2e965daf3158d7f1846f1306305f41bef",
-        "state_root": "0xfd18cf40cc907a739be483f1ca0ee23ad65cdd3df23205eabc6d660a75d1f54e",
-        "body_root": "0x5a75dcd745c46d94e1b52a2d5834087ae5fae810fa4d69ee800f3753fc05a4b2"
+        "proposer_index": "4565872325553958186",
+        "parent_root": "0x77d96e3f4a4ad0971596b71d6420b24b4d12a275af3d948b77b438faa484f0d1",
+        "state_root": "0x8afa683fea95afdc0ad91e4937a9c6185315a1076506bd45a4357cc27fe5a75c",
+        "body_root": "0x228e536ac5e8e14792142faf9de3db385988682e71222198cb3a108078b686d7"
       },
       "signature": "0x8ab48d0165e8cc8bfc6804760141896946c3be751af5f16d1a03ead456ebceab88a8168e1da194df56c512edc15ca6350c413cb819d31a7ef69b329d8cb9fa8e5d93ae66289a445f1465d57d97e72d4c0866e48a806df7f58ebcd060e2d03a72"
     }

data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/getBlockHeaders.json

@@ -3,29 +3,29 @@
   "finalized": false,
   "data": [
     {
-      "root": "0x127698f65207baf45ec6497931e82df707176d84588b76f6f1e4c4c577de3b94",
+      "root": "0x9042b435b163854b76508411d10bb170123cbe59563b862fac9d25d7493c11a9",
       "canonical": true,
       "header": {
         "message": {
           "slot": "1",
-          "proposer_index": "4666673844721362956",
-          "parent_root": "0x367cbd40ac7318427aadb97345a91fa2e965daf3158d7f1846f1306305f41bef",
-          "state_root": "0xfd18cf40cc907a739be483f1ca0ee23ad65cdd3df23205eabc6d660a75d1f54e",
-          "body_root": "0x5a75dcd745c46d94e1b52a2d5834087ae5fae810fa4d69ee800f3753fc05a4b2"
+          "proposer_index": "4565872325553958186",
+          "parent_root": "0x77d96e3f4a4ad0971596b71d6420b24b4d12a275af3d948b77b438faa484f0d1",
+          "state_root": "0x8afa683fea95afdc0ad91e4937a9c6185315a1076506bd45a4357cc27fe5a75c",
+          "body_root": "0x228e536ac5e8e14792142faf9de3db385988682e71222198cb3a108078b686d7"
         },
         "signature": "0x8ab48d0165e8cc8bfc6804760141896946c3be751af5f16d1a03ead456ebceab88a8168e1da194df56c512edc15ca6350c413cb819d31a7ef69b329d8cb9fa8e5d93ae66289a445f1465d57d97e72d4c0866e48a806df7f58ebcd060e2d03a72"
       }
     },
     {
-      "root": "0xdcaa6769bd88a5fc1302760cc1f20fe57d04f6164335cdf2149e54d58402d473",
+      "root": "0xe8c2353c598768e5a084180f7907f784dfa456db0ccf204b28880c36f4ced788",
       "canonical": true,
       "header": {
         "message": {
           "slot": "1",
-          "proposer_index": "4559262388392254378",
-          "parent_root": "0xc35d573fca784dabeaa154cbb2430480661e9ebd886037742eb9461b0e08cefc",
-          "state_root": "0xd67e513f6ac42cf0dfe4bbf686cc184d6c219d4f3e29602e5b3a8ae3e8688587",
-          "body_root": "0x6f8f410666c7a0d861b9984c17d50ddb50f7ec7bef63b48c43ad6695a12b55df"
+          "proposer_index": "4888106694541283922",
+          "parent_root": "0x372cd043f2dd36351da1acbb7f0a6049a4d05e88a37d26a75153db8fb6411f53",
+          "state_root": "0x96d1b2431158938de8efb094a1b6c64ac3df59623369f24936d92c79fa25b408",
+          "body_root": "0x9727a900c69c3b5a1afd4c0dcc2a9e3282d4890587f5871a8071c941637a51ab"
         },
         "signature": "0xa2695a1a43803793a6a1e80b15798147d8a63da5c8a6ee3e18dabd190255f48cbe36c33e3ffa372e11b19605cd2282a3136a2662cd3c8f9d3d0345ec1593b4241a0ee66c0d626da3db5cc6bd7d1119063922633c1c2987d69839af6ac37eb675"
       }

data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v1/builder/getExpectedWithdrawals.json

@@ -3,10 +3,10 @@
   "finalized": false,
   "data": [
     {
-      "index": "4503077933255190248",
-      "validator_index": "2357271",
-      "address": "0x42fb7d43b1006df9874a521b12867f80fbfa5084",
-      "amount": "4868276887351139793"
+      "index": "4478290672120026440",
+      "validator_index": "2149275",
+      "address": "0xb4b9be3e0927fba9d26ed13331291a54096d8491",
+      "amount": "4519602774011966121"
     }
   ]
 }

ethereum/spec/src/test/java/tech/pegasys/teku/spec/signatures/LocalSignerTest.java

@@ -54,7 +54,7 @@ public void shouldSignBlock() {
     final BLSSignature expectedSignature =
         BLSSignature.fromBytesCompressed(
             Bytes.fromBase64String(
-                "owmSdVC0ohD8KseJ5xXVI8MrX+sJkrWJf7OMBbAJm4cugs0idsnksufjWdPmmtPDCgklh+oxhXxvNB32NkWMgS1WKBtC0/Ia4POxdURWx3klGx6w3og+yQGlPG/rJw2x"));
+                "o6/J6sxThPzADTjmP8T6tuJvG+Rll8usMxCCbq5sMItDeblgqsPXgbPihBDcJo0pBEpPZec1uPPd9rNK9O4uKhIuXw2rKzFcYzLvOjS50kpNHVQ42HN8gUmcmvC06tLo"));
 
     final SafeFuture<BLSSignature> result = signer.signBlock(block, fork);
     asyncRunner.executeQueuedActions();

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.spec.util;
 
 import static com.google.common.base.Preconditions.checkState;
+import static ethereum.ckzg4844.CKZG4844JNI.BYTES_PER_CELL;
 import static java.util.stream.Collectors.toList;
 import static tech.pegasys.teku.ethereum.pow.api.DepositConstants.DEPOSIT_CONTRACT_TREE_DEPTH;
 import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ZERO;
@@ -77,6 +78,7 @@
 import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszPrimitiveVectorSchema;
 import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszUInt64ListSchema;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.kzg.KZGCell;
 import tech.pegasys.teku.kzg.KZGCommitment;
 import tech.pegasys.teku.kzg.KZGProof;
 import tech.pegasys.teku.spec.Spec;
@@ -92,6 +94,12 @@
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.Cell;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.CellSchema;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumn;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumnSchema;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumnSidecar;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumnSidecarSchema;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockAndState;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader;
@@ -195,12 +203,14 @@
 import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes;
 import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb;
 import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash;
+import tech.pegasys.teku.spec.logic.versions.fulu.helpers.MiscHelpersFulu;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitions;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsAltair;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsBellatrix;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsCapella;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra;
+import tech.pegasys.teku.spec.schemas.SchemaDefinitionsFulu;
 
 public final class DataStructureUtil {
 
@@ -1124,10 +1134,13 @@ public BeaconBlock randomBeaconBlock(final long slotNum) {
   }
 
   public BeaconBlock randomBeaconBlock(final UInt64 slotNum) {
+    return randomBeaconBlock(slotNum, randomBeaconBlockBody(slotNum));
+  }
+
+  public BeaconBlock randomBeaconBlock(final UInt64 slotNum, final BeaconBlockBody body) {
     final UInt64 proposerIndex = randomUInt64();
     final Bytes32 previousRoot = randomBytes32();
     final Bytes32 stateRoot = randomBytes32();
-    final BeaconBlockBody body = randomBeaconBlockBody(slotNum);
 
     return new BeaconBlock(
         spec.atSlot(slotNum).getSchemaDefinitions().getBeaconBlockSchema(),
@@ -2551,6 +2564,131 @@ public BlobSidecar build() {
     }
   }
 
+  public class RandomSidecarBuilder {
+    private Optional<UInt64> index = Optional.empty();
+    private Optional<DataColumn> dataColumn = Optional.empty();
+    private Optional<List<KZGCommitment>> kzgCommitments = Optional.empty();
+    private Optional<List<KZGProof>> kzgProofs = Optional.empty();
+    private Optional<SignedBeaconBlockHeader> signedBeaconBlockHeader = Optional.empty();
+    private Optional<List<Bytes32>> kzgCommitmentsInclusionProof = Optional.empty();
+
+    public RandomSidecarBuilder index(final UInt64 index) {
+      this.index = Optional.of(index);
+      return this;
+    }
+
+    public RandomSidecarBuilder dataColumn(final DataColumn dataColumn) {
+      this.dataColumn = Optional.of(dataColumn);
+      return this;
+    }
+
+    public RandomSidecarBuilder kzgCommitments(final List<KZGCommitment> kzgCommitments) {
+      this.kzgCommitments = Optional.of(kzgCommitments);
+      return this;
+    }
+
+    public RandomSidecarBuilder kzgProofs(final List<KZGProof> kzgProofs) {
+      this.kzgProofs = Optional.of(kzgProofs);
+      return this;
+    }
+
+    public RandomSidecarBuilder signedBeaconBlockHeader(
+        final SignedBeaconBlockHeader signedBeaconBlockHeader) {
+      this.signedBeaconBlockHeader = Optional.of(signedBeaconBlockHeader);
+      return this;
+    }
+
+    public RandomSidecarBuilder kzgCommitmentsInclusionProof(
+        final List<Bytes32> kzgCommitmentsInclusionProof) {
+      this.kzgCommitmentsInclusionProof = Optional.of(kzgCommitmentsInclusionProof);
+      return this;
+    }
+
+    public DataColumnSidecar build() {
+      final SignedBeaconBlockHeader signedBlockHeader =
+          signedBeaconBlockHeader.orElseGet(DataStructureUtil.this::randomSignedBeaconBlockHeader);
+      final DataColumnSidecarSchema dataColumnSidecarSchema =
+          getFuluSchemaDefinitions(signedBlockHeader.getMessage().getSlot())
+              .getDataColumnSidecarSchema();
+      final int numberOfProofs =
+          kzgProofs
+              .map(List::size)
+              .or(() -> kzgCommitments.map(List::size))
+              .orElseGet(DataStructureUtil.this::randomNumberOfBlobsPerBlock);
+
+      return dataColumnSidecarSchema.create(
+          index.orElseGet(DataStructureUtil.this::randomDataColumnSidecarIndex),
+          dataColumn.orElseGet(
+              () -> randomDataColumn(signedBlockHeader.getMessage().getSlot(), numberOfProofs)),
+          kzgCommitments.orElseGet(
+              () ->
+                  IntStream.range(0, numberOfProofs)
+                      .mapToObj(__ -> randomKZGCommitment())
+                      .toList()),
+          kzgProofs.orElseGet(
+              () -> IntStream.range(0, numberOfProofs).mapToObj(__ -> randomKZGProof()).toList()),
+          signedBlockHeader,
+          kzgCommitmentsInclusionProof.orElseGet(
+              () ->
+                  IntStream.range(
+                          0,
+                          dataColumnSidecarSchema
+                              .getKzgCommitmentsInclusionProofSchema()
+                              .getLength())
+                      .mapToObj(__ -> randomBytes32())
+                      .toList()));
+    }
+  }
+
+  public DataColumn randomDataColumn(final UInt64 slot, final int blobs) {
+    final DataColumnSchema dataColumnSchema = getFuluSchemaDefinitions(slot).getDataColumnSchema();
+    List<Cell> list = IntStream.range(0, blobs).mapToObj(__ -> randomCell(slot)).toList();
+    return dataColumnSchema.create(list);
+  }
+
+  public Cell randomCell(final UInt64 slot) {
+    final CellSchema cellSchema = getFuluSchemaDefinitions(slot).getCellSchema();
+    return cellSchema.create(randomBytes(cellSchema.getLength()));
+  }
+
+  public KZGCell randomKZGCell() {
+    return new KZGCell(randomBytes(BYTES_PER_CELL));
+  }
+
+  public DataColumnSidecar randomDataColumnSidecar() {
+    return new RandomSidecarBuilder().build();
+  }
+
+  public DataColumnSidecar randomDataColumnSidecar(
+      final SignedBeaconBlockHeader header, final UInt64 index) {
+    return new RandomSidecarBuilder().signedBeaconBlockHeader(header).index(index).build();
+  }
+
+  public DataColumnSidecar randomDataColumnSidecarWithInclusionProof(
+      final SignedBeaconBlock signedBeaconBlock, final UInt64 index) {
+    final MiscHelpersFulu miscHelpersFulu =
+        MiscHelpersFulu.required(spec.getGenesisSpec().miscHelpers());
+    final List<KZGCommitment> kzgCommitments =
+        signedBeaconBlock
+            .getMessage()
+            .getBody()
+            .getOptionalBlobKzgCommitments()
+            .orElseThrow()
+            .asList()
+            .stream()
+            .map(SszKZGCommitment::getKZGCommitment)
+            .toList();
+    final List<Bytes32> inclusionProof =
+        miscHelpersFulu.computeDataColumnKzgCommitmentsInclusionProof(
+            signedBeaconBlock.getBeaconBlock().orElseThrow().getBody());
+    return new RandomSidecarBuilder()
+        .signedBeaconBlockHeader(signedBeaconBlock.asHeader())
+        .kzgCommitments(kzgCommitments)
+        .kzgCommitmentsInclusionProof(inclusionProof)
+        .index(index)
+        .build();
+  }
+
   public List<Bytes32> randomKzgCommitmentInclusionProof() {
     final int depth =
         SpecConfigDeneb.required(spec.forMilestone(SpecMilestone.DENEB).getConfig())
@@ -2684,6 +2822,15 @@ private int randomNumberOfBlobsPerBlock() {
             + 1);
   }
 
+  public UInt64 randomDataColumnSidecarIndex() {
+    return randomUInt64(
+        spec.forMilestone(spec.getForkSchedule().getHighestSupportedMilestone())
+            .getConfig()
+            .toVersionFulu()
+            .orElseThrow()
+            .getNumberOfColumns());
+  }
+
   private int randomInt(final int origin, final int bound) {
     return new Random(nextSeed()).ints(origin, bound).findFirst().orElse(0);
   }
@@ -2712,6 +2859,10 @@ private SchemaDefinitionsElectra getElectraSchemaDefinitions(final UInt64 slot)
     return SchemaDefinitionsElectra.required(spec.atSlot(slot).getSchemaDefinitions());
   }
 
+  private SchemaDefinitionsFulu getFuluSchemaDefinitions(final UInt64 slot) {
+    return SchemaDefinitionsFulu.required(spec.atSlot(slot).getSchemaDefinitions());
+  }
+
   int getEpochsPerEth1VotingPeriod() {
     return getConstant(SpecConfig::getEpochsPerEth1VotingPeriod);
   }

ethereum/statetransition/build.gradle

@@ -50,6 +50,7 @@ dependencies {
   testFixturesImplementation project(':infrastructure:time')
   testFixturesImplementation testFixtures(project(':ethereum:spec'))
   testFixturesImplementation testFixtures(project(':infrastructure:metrics'))
+  testFixturesImplementation testFixtures(project(':infrastructure:time'))
   testImplementation testFixtures(project(':infrastructure:logging'))
 
   jmhImplementation testFixtures(project(':infrastructure:bls'))

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/datacolumns/MinCustodyPeriodSlotCalculator.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.statetransition.datacolumns;
+
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.config.SpecConfigFulu;
+
+public interface MinCustodyPeriodSlotCalculator {
+
+  static MinCustodyPeriodSlotCalculator createFromSpec(final Spec spec) {
+    return currentSlot -> {
+      final UInt64 currentEpoch = spec.computeEpochAtSlot(currentSlot);
+      final int custodyPeriodEpochs =
+          spec.getSpecConfig(currentEpoch)
+              .toVersionFulu()
+              .map(SpecConfigFulu::getMinEpochsForDataColumnSidecarsRequests)
+              .orElse(0);
+      if (custodyPeriodEpochs == 0) {
+        return currentSlot;
+      } else {
+        final UInt64 minCustodyEpoch = currentEpoch.minusMinZero(custodyPeriodEpochs);
+        return spec.computeStartSlotAtEpoch(minCustodyEpoch);
+      }
+    };
+  }
+
+  UInt64 getMinCustodyPeriodSlot(UInt64 currentSlot);
+}