Updated MiscHelpersFulu + Added Gossip Logger

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/deneb/merkle_proof/SingleMerkleProofTestExecutor.java

@@ -32,10 +32,13 @@
 import tech.pegasys.teku.reference.TestDataUtils;
 import tech.pegasys.teku.reference.TestExecutor;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
+import tech.pegasys.teku.spec.config.SpecConfigFulu;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody;
 import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
 import tech.pegasys.teku.spec.logic.common.helpers.Predicates;
 import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb;
+import tech.pegasys.teku.spec.logic.versions.fulu.helpers.MiscHelpersFulu;
+import tech.pegasys.teku.spec.schemas.SchemaDefinitionsFulu;
 
 public class SingleMerkleProofTestExecutor implements TestExecutor {
   private static final Pattern TEST_NAME_PATTERN = Pattern.compile("(.+)/(.+)");
@@ -88,9 +91,12 @@ void runBeaconBlockBodyTest(
             testDefinition,
             OBJECT_SSZ_FILE,
             testDefinition.getSpec().getGenesisSchemaDefinitions().getBeaconBlockBodySchema());
-
+    // Deneb
     if (proofType.startsWith("blob_kzg_commitment_merkle_proof")) {
       runBlobKzgCommitmentMerkleProofTest(testDefinition, data, beaconBlockBody);
+      // Fulu
+    } else if (proofType.startsWith("blob_kzg_commitments_merkle_proof")) {
+      runBlobKzgCommitmentsMerkleProofTest(testDefinition, data, beaconBlockBody);
     } else {
       throw new RuntimeException("Unknown proof type " + proofType);
     }
@@ -130,11 +136,35 @@ private void runBlobKzgCommitmentMerkleProofTest(
     assertThat(miscHelpersDeneb.getBlobSidecarKzgCommitmentGeneralizedIndex(kzgCommitmentIndex))
         .isEqualTo(data.leafIndex);
     assertThat(
-            miscHelpersDeneb.computeKzgCommitmentInclusionProof(
+            miscHelpersDeneb.computeBlobKzgCommitmentInclusionProof(
                 kzgCommitmentIndex, beaconBlockBody))
         .isEqualTo(data.branch.stream().map(Bytes32::fromHexString).toList());
   }
 
+  private void runBlobKzgCommitmentsMerkleProofTest(
+      final TestDefinition testDefinition, final Data data, final BeaconBlockBody beaconBlockBody) {
+    final Predicates predicates = new Predicates(testDefinition.getSpec().getGenesisSpecConfig());
+    final Bytes32 kzgCommitmentsHash = Bytes32.fromHexString(data.leaf);
+
+    // Forward check
+    assertThat(
+            predicates.isValidMerkleBranch(
+                kzgCommitmentsHash,
+                createKzgCommitmentsMerkleProofBranchFromData(testDefinition, data.branch),
+                getKzgCommitmentsInclusionProofDepth(testDefinition),
+                data.leafIndex,
+                beaconBlockBody.hashTreeRoot()))
+        .isTrue();
+
+    // Verify 2 MiscHelpersFulu helpers
+    final MiscHelpersFulu miscHelpersFulu =
+        MiscHelpersFulu.required(testDefinition.getSpec().getGenesisSpec().miscHelpers());
+    assertThat(miscHelpersFulu.getBlockBodyKzgCommitmentsGeneralizedIndex())
+        .isEqualTo(data.leafIndex);
+    assertThat(miscHelpersFulu.computeDataColumnKzgCommitmentsInclusionProof(beaconBlockBody))
+        .isEqualTo(data.branch.stream().map(Bytes32::fromHexString).toList());
+  }
+
   private SszBytes32Vector createKzgCommitmentMerkleProofBranchFromData(
       final TestDefinition testDefinition, final List<String> branch) {
     final SszBytes32VectorSchema<?> kzgCommitmentInclusionProofSchema =
@@ -153,4 +183,20 @@ private int getKzgCommitmentInclusionProofDepth(final TestDefinition testDefinit
     return SpecConfigDeneb.required(testDefinition.getSpec().getGenesisSpecConfig())
         .getKzgCommitmentInclusionProofDepth();
   }
+
+  private SszBytes32Vector createKzgCommitmentsMerkleProofBranchFromData(
+      final TestDefinition testDefinition, final List<String> branch) {
+    final SszBytes32VectorSchema<?> kzgCommitmentsInclusionProofSchema =
+        SchemaDefinitionsFulu.required(testDefinition.getSpec().getGenesisSchemaDefinitions())
+            .getDataColumnSidecarSchema()
+            .getKzgCommitmentsInclusionProofSchema();
+    return kzgCommitmentsInclusionProofSchema.createFromElements(
+        branch.stream().map(Bytes32::fromHexString).map(SszBytes32::of).toList());
+  }
+
+  private int getKzgCommitmentsInclusionProofDepth(final TestDefinition testDefinition) {
+    return SpecConfigFulu.required(testDefinition.getSpec().getGenesisSpecConfig())
+        .getKzgCommitmentsInclusionProofDepth()
+        .intValue();
+  }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -35,8 +35,6 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import javax.annotation.CheckReturnValue;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSPublicKey;
@@ -54,10 +52,12 @@
 import tech.pegasys.teku.spec.config.SpecConfigAltair;
 import tech.pegasys.teku.spec.config.SpecConfigAndParent;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
+import tech.pegasys.teku.spec.config.SpecConfigFulu;
 import tech.pegasys.teku.spec.constants.Domain;
 import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumnSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockAndState;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader;
@@ -108,7 +108,6 @@
 import tech.pegasys.teku.spec.schemas.registry.SchemaRegistryBuilder;
 
 public class Spec {
-  private static final Logger LOG = LogManager.getLogger();
   private final Map<SpecMilestone, SpecVersion> specVersions;
   private final ForkSchedule forkSchedule;
   private final StateTransition stateTransition;
@@ -147,13 +146,6 @@ static Spec create(
 
     final ForkSchedule forkSchedule = forkScheduleBuilder.build();
 
-    final UInt64 lastForkActivationEpoch =
-        forkSchedule.getActiveMilestones().getLast().getFork().getEpoch();
-    LOG.info(
-        "Creating network specification. Highest milestone supported: {}, from epoch {}",
-        highestMilestoneSupported.lowerCaseName(),
-        lastForkActivationEpoch);
-
     return new Spec(specConfigAndParent, specVersions, forkSchedule);
   }
 
@@ -433,6 +425,16 @@ public ExecutionPayloadHeader deserializeJsonExecutionPayloadHeader(
         .jsonDeserialize(objectMapper.createParser(jsonFile));
   }
 
+  public DataColumnSidecar deserializeSidecar(final Bytes serializedSidecar, final UInt64 slot) {
+    return atSlot(slot)
+        .getSchemaDefinitions()
+        .toVersionFulu()
+        .orElseThrow(
+            () -> new RuntimeException("FULU milestone is required to deserialize column sidecar"))
+        .getDataColumnSidecarSchema()
+        .sszDeserialize(serializedSidecar);
+  }
+
   // BeaconState
   public UInt64 getCurrentEpoch(final BeaconState state) {
     return atState(state).beaconStateAccessors().getCurrentEpoch(state);
@@ -948,14 +950,14 @@ public boolean isAvailabilityOfBlobSidecarsRequiredAtSlot(
 
   public boolean isAvailabilityOfBlobSidecarsRequiredAtEpoch(
       final ReadOnlyStore store, final UInt64 epoch) {
-    if (!forkSchedule.getSpecMilestoneAtEpoch(epoch).isGreaterThanOrEqualTo(DENEB)) {
-      return false;
-    }
-    final SpecConfig config = atEpoch(epoch).getConfig();
-    final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(config);
-    return getCurrentEpoch(store)
-        .minusMinZero(epoch)
-        .isLessThanOrEqualTo(specConfigDeneb.getMinEpochsForBlobSidecarsRequests());
+    return atEpoch(epoch)
+        .miscHelpers()
+        .toVersionDeneb()
+        .map(
+            denebMiscHelpers ->
+                denebMiscHelpers.isAvailabilityOfBlobSidecarsRequiredAtEpoch(
+                    getCurrentEpoch(store), epoch))
+        .orElse(false);
   }
 
   /**
@@ -1000,6 +1002,26 @@ public UInt64 computeSubnetForBlobSidecar(final BlobSidecar blobSidecar) {
                 .getBlobSidecarSubnetCount());
   }
 
+  public Optional<Integer> getNumberOfDataColumns() {
+    return getSpecConfigFulu().map(SpecConfigFulu::getNumberOfColumns);
+  }
+
+  public Optional<Integer> getNumberOfDataColumnSubnets() {
+    return getSpecConfigFulu().map(SpecConfigFulu::getDataColumnSidecarSubnetCount);
+  }
+
+  public boolean isAvailabilityOfDataColumnSidecarsRequiredAtEpoch(
+      final ReadOnlyStore store, final UInt64 epoch) {
+    if (getSpecConfigFulu().isEmpty()) {
+      return false;
+    }
+    final SpecConfig config = atEpoch(epoch).getConfig();
+    final SpecConfigFulu specConfigFulu = SpecConfigFulu.required(config);
+    return getCurrentEpoch(store)
+        .minusMinZero(epoch)
+        .isLessThanOrEqualTo(specConfigFulu.getMinEpochsForDataColumnSidecarsRequests());
+  }
+
   public Optional<UInt64> computeFirstSlotWithBlobSupport() {
     return getSpecConfigDeneb()
         .map(SpecConfigDeneb::getDenebForkEpoch)
@@ -1020,6 +1042,15 @@ private Optional<SpecConfigDeneb> getSpecConfigDeneb() {
         .flatMap(SpecConfig::toVersionDeneb);
   }
 
+  // Fulu private helpers
+  private Optional<SpecConfigFulu> getSpecConfigFulu() {
+    final SpecMilestone highestSupportedMilestone =
+        getForkSchedule().getHighestSupportedMilestone();
+    return Optional.ofNullable(forMilestone(highestSupportedMilestone))
+        .map(SpecVersion::getConfig)
+        .flatMap(SpecConfig::toVersionFulu);
+  }
+
   // Private helpers
   private SpecVersion atState(final BeaconState state) {
     return atSlot(state.getSlot());

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndCellProofs.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.spec.datastructures.execution;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.kzg.KZG.CELLS_PER_EXT_BLOB;
+
+import java.util.List;
+import tech.pegasys.teku.kzg.KZGProof;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob;
+
+public record BlobAndCellProofs(Blob blob, List<KZGProof> cellProofs) {
+  public BlobAndCellProofs(final Blob blob, final List<KZGProof> cellProofs) {
+    checkArgument(
+        cellProofs.size() == CELLS_PER_EXT_BLOB,
+        "Expected %s proofs but got %s",
+        CELLS_PER_EXT_BLOB,
+        cellProofs.size());
+    this.blob = blob;
+    this.cellProofs = cellProofs;
+  }
+}

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MathHelpers.java

@@ -18,6 +18,7 @@
 import java.nio.ByteOrder;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.units.bigints.UInt256;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 
 public class MathHelpers {
@@ -134,4 +135,20 @@ static Bytes32 uintToBytes32(final UInt64 value) {
   public static UInt64 bytesToUInt64(final Bytes data) {
     return UInt64.fromLongBits(data.toLong(ByteOrder.LITTLE_ENDIAN));
   }
+
+  public static Bytes uint256ToBytes(final UInt256 number) {
+    final Bytes intBytes =
+        Bytes.wrap(number.toUnsignedBigInteger(ByteOrder.LITTLE_ENDIAN).toByteArray())
+            .trimLeadingZeros();
+    // We should keep 32 bytes
+    return Bytes32.leftPad(intBytes);
+  }
+
+  public static int intPlusMaxIntCapped(final int a, final int b) {
+    final UInt64 sum = UInt64.valueOf(a).plus(b);
+    if (sum.isLessThanOrEqualTo(UInt64.valueOf(Integer.MAX_VALUE))) {
+      return sum.intValue();
+    }
+    return Integer.MAX_VALUE;
+  }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java

@@ -217,7 +217,7 @@ public List<UInt64> computeSubscribedSubnets(final UInt256 nodeId, final UInt64
         .toList();
   }
 
-  private UInt64 computeSubscribedSubnet(
+  protected UInt64 computeSubscribedSubnet(
       final UInt256 nodeId, final UInt64 epoch, final int index) {
 
     final int nodeIdPrefix =

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java

@@ -55,6 +55,7 @@ public class MiscHelpersDeneb extends MiscHelpersCapella {
   private final Predicates predicates;
   private final BeaconBlockBodySchemaDeneb<?> beaconBlockBodySchema;
   private final BlobSidecarSchema blobSidecarSchema;
+  private final SpecConfigDeneb specConfigDeneb;
 
   public static MiscHelpersDeneb required(final MiscHelpers miscHelpers) {
     return miscHelpers
@@ -71,6 +72,7 @@ public MiscHelpersDeneb(
       final Predicates predicates,
       final SchemaDefinitionsDeneb schemaDefinitions) {
     super(specConfig);
+    this.specConfigDeneb = specConfig;
     this.predicates = predicates;
     this.beaconBlockBodySchema =
         (BeaconBlockBodySchemaDeneb<?>) schemaDefinitions.getBeaconBlockBodySchema();
@@ -240,7 +242,7 @@ public int getBlobSidecarKzgCommitmentGeneralizedIndex(final UInt64 blobSidecarI
         GIndexUtil.gIdxCompose(blobKzgCommitmentsGeneralizedIndex, commitmentGeneralizedIndex);
   }
 
-  public List<Bytes32> computeKzgCommitmentInclusionProof(
+  public List<Bytes32> computeBlobKzgCommitmentInclusionProof(
       final UInt64 blobSidecarIndex, final BeaconBlockBody beaconBlockBody) {
     return MerkleUtil.constructMerkleProof(
         beaconBlockBody.getBackingNode(),
@@ -265,7 +267,7 @@ public BlobSidecar constructBlobSidecar(
               index, commitmentsCount));
     }
     final List<Bytes32> kzgCommitmentInclusionProof =
-        computeKzgCommitmentInclusionProof(index, beaconBlockBody);
+        computeBlobKzgCommitmentInclusionProof(index, beaconBlockBody);
     return blobSidecarSchema.create(
         index, blob, commitment, proof, signedBeaconBlock.asHeader(), kzgCommitmentInclusionProof);
   }
@@ -286,7 +288,8 @@ public BlobSidecar constructBlobSidecarFromBlobAndProof(
             sszKZGCommitment,
             new SszKZGProof(blobAndProof.proof()),
             signedBeaconBlockHeader,
-            computeKzgCommitmentInclusionProof(blobIdentifier.getIndex(), beaconBlockBodyDeneb));
+            computeBlobKzgCommitmentInclusionProof(
+                blobIdentifier.getIndex(), beaconBlockBodyDeneb));
 
     blobSidecar.markSignatureAsValidated();
     blobSidecar.markKzgCommitmentInclusionProofAsValidated();
@@ -314,4 +317,11 @@ public boolean verifyBlobKzgCommitmentInclusionProof(final BlobSidecar blobSidec
 
     return result;
   }
+
+  public boolean isAvailabilityOfBlobSidecarsRequiredAtEpoch(
+      final UInt64 currentEpoch, final UInt64 epoch) {
+    return currentEpoch
+        .minusMinZero(epoch)
+        .isLessThanOrEqualTo(specConfigDeneb.getMinEpochsForBlobSidecarsRequests());
+  }
 }