Enable gossip attestation and aggregate ref-test

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/TestDataUtils.java

@@ -25,6 +25,7 @@
 import java.util.Optional;
 import java.util.function.Function;
 import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
 import org.xerial.snappy.Snappy;
 import org.yaml.snakeyaml.LoaderOptions;
 import tech.pegasys.teku.ethtests.finder.TestDefinition;
@@ -149,12 +150,28 @@ public static AnchorPoint createAnchorFromState(final Spec spec, final BeaconSta
    */
   public static AnchorPoint createAnchorFromStateAndMatchingBlock(
       final Spec spec, final BeaconState state, final Collection<SignedBeaconBlock> blocks) {
+    final Optional<AnchorPoint> anchorFromMatchingStateRoot =
+        blocks.stream()
+            .filter(block -> block.getStateRoot().equals(state.hashTreeRoot()))
+            .findFirst()
+            .map(
+                block ->
+                    AnchorPoint.fromInitialBlockAndState(
+                        spec, new SignedBlockAndState(block, state)));
+    if (anchorFromMatchingStateRoot.isPresent()) {
+      return anchorFromMatchingStateRoot.get();
+    }
+
+    final Bytes32 stateBlockRoot = BeaconBlockHeader.fromState(state).hashTreeRoot();
     return blocks.stream()
-        .filter(block -> block.getStateRoot().equals(state.hashTreeRoot()))
+        .filter(block -> block.getRoot().equals(stateBlockRoot))
         .findFirst()
         .map(
-            block ->
-                AnchorPoint.fromInitialBlockAndState(spec, new SignedBlockAndState(block, state)))
+            block -> {
+              final UInt64 epoch = spec.computeNextEpochBoundary(state.getSlot());
+              final Checkpoint checkpoint = new Checkpoint(epoch, block.getRoot());
+              return AnchorPoint.create(spec, checkpoint, state, Optional.of(block));
+            })
         .orElseGet(() -> createAnchorFromState(spec, state));
   }
 }

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/gossip/GossipBeaconAggregateAndProofTestExecutor.java

@@ -22,10 +22,10 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
 import tech.pegasys.teku.ethtests.finder.TestDefinition;
@@ -39,6 +39,7 @@
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.operations.SignedAggregateAndProof;
 import tech.pegasys.teku.spec.datastructures.state.AnchorPoint;
+import tech.pegasys.teku.spec.datastructures.state.Checkpoint;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannelStub;
 import tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult;
@@ -60,6 +61,7 @@
 import tech.pegasys.teku.storage.server.StateStorageMode;
 import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder;
 import tech.pegasys.teku.storage.storageSystem.StorageSystem;
+import tech.pegasys.teku.storage.store.UpdatableStore;
 
 public class GossipBeaconAggregateAndProofTestExecutor implements TestExecutor {
 
@@ -124,24 +126,21 @@ public void runTest(final TestDefinition testDefinition) throws Throwable {
 
     forkChoice.onTick(UInt64.valueOf(metaData.getCurrentTimeMs()), Optional.empty());
 
-    // Track block roots that explicitly failed validation (marked failed: true in meta.yaml).
-    // We load these blocks to obtain their hash tree root but do not import them, mirroring the
-    // spec distinction between "block not seen" (IGNORE) and "block failed validation" (REJECT).
-    final Set<Bytes32> failedBlockRoots = new HashSet<>();
+    // Blocks marked failed: true in meta.yaml are recorded as invalid (by root) and not imported,
+    // mirroring the invalidBlockRoots map maintained in production by BlockManager. The attestation
+    // validator rejects aggregates voting for one of these roots.
+    final Map<Bytes32, BlockImportResult> invalidBlockRoots = new HashMap<>();
 
-    // When a custom finalized_checkpoint is set, its root will not be in the chain, so any block
-    // import attempt would fail. Skip all imports so that the validator's block-not-available path
-    // produces the expected IGNORE result.
-    final boolean hasCustomFinalizedCheckpoint = metaData.getFinalizedCheckpoint() != null;
     for (final BlockEntryAndBlock blockEntryAndBlock : blocks) {
       final GossipBeaconAggregateAndProofMetaData.BlockEntry blockEntry =
           blockEntryAndBlock.blockEntry();
       final SignedBeaconBlock block = blockEntryAndBlock.block();
       if (blockEntry.isFailed()) {
         // Record the root of this invalid block so aggregates voting for it are rejected.
         // Don't import it — a NOOP BLS verifier would accept it despite the bad signature.
-        failedBlockRoots.add(block.getRoot());
-      } else if (!hasCustomFinalizedCheckpoint && !block.getRoot().equals(anchorPoint.getRoot())) {
+        invalidBlockRoots.put(
+            block.getRoot(), BlockImportResult.FAILED_DESCENDANT_OF_INVALID_BLOCK);
+      } else if (!block.getRoot().equals(anchorPoint.getRoot())) {
         final BlockImportResult importResult =
             safeJoin(
                 forkChoice.onBlock(
@@ -152,11 +151,28 @@ public void runTest(final TestDefinition testDefinition) throws Throwable {
       }
     }
 
+    Optional<Checkpoint> customFinalizedCheckpoint = Optional.empty();
+    if (metaData.getFinalizedCheckpoint() != null) {
+      final GossipBeaconAggregateAndProofMetaData.FinalizedCheckpoint finalizedCheckpoint =
+          metaData.getFinalizedCheckpoint();
+      if (finalizedCheckpoint.getBlock() != null) {
+        final Checkpoint checkpoint = finalizedCheckpoint.toCheckpoint(testDefinition, spec);
+        final UpdatableStore.StoreTransaction tx = recentChainData.startStoreTransaction();
+        tx.setFinalizedCheckpoint(checkpoint, false);
+        safeJoin(tx.commit());
+      } else {
+        customFinalizedCheckpoint =
+            Optional.of(finalizedCheckpoint.toCheckpoint(testDefinition, spec));
+      }
+    }
+
     final AttestationValidator attestationValidator =
         new AttestationValidator(
             spec,
             AsyncBLSSignatureVerifier.wrap(blsVerifier),
-            new GossipValidationHelper(spec, recentChainData, metricsSystem));
+            createGossipValidationHelper(
+                spec, recentChainData, metricsSystem, customFinalizedCheckpoint),
+            invalidBlockRoots);
     final AggregateAttestationValidator aggregateValidator =
         new AggregateAttestationValidator(
             spec, attestationValidator, AsyncBLSSignatureVerifier.wrap(blsVerifier));
@@ -172,18 +188,6 @@ public void runTest(final TestDefinition testDefinition) throws Throwable {
               message.getMessage() + ".ssz_snappy",
               spec.getGenesisSchemaDefinitions().getSignedAggregateAndProofSchema());
 
-      // Failed-block check: aggregate votes for a block that failed validation
-      final Bytes32 votedBlockRoot =
-          signedAggregateAndProof.getMessage().getAggregate().getData().getBeaconBlockRoot();
-      if (failedBlockRoots.contains(votedBlockRoot)) {
-        assertThat(message.getExpected())
-            .describedAs(
-                "Expected reject for aggregate %s voting for failed block %s",
-                message.getMessage(), votedBlockRoot)
-            .isEqualTo("reject");
-        continue;
-      }
-
       final ValidatableAttestation validatableAttestation =
           ValidatableAttestation.aggregateFromNetwork(spec, signedAggregateAndProof);
       final InternalValidationResult result =
@@ -218,6 +222,29 @@ public void runTest(final TestDefinition testDefinition) throws Throwable {
     }
   }
 
+  private static GossipValidationHelper createGossipValidationHelper(
+      final Spec spec,
+      final RecentChainData recentChainData,
+      final StubMetricsSystem metricsSystem,
+      final Optional<Checkpoint> finalizedCheckpointOverride) {
+    return finalizedCheckpointOverride
+        .<GossipValidationHelper>map(
+            finalizedCheckpoint ->
+                new GossipValidationHelper(spec, recentChainData, metricsSystem) {
+                  @Override
+                  public boolean currentFinalizedCheckpointIsAncestorOfAttestationBlock(
+                      final Bytes32 blockRoot) {
+                    return spec.getAncestor(
+                            getForkChoiceStrategy(),
+                            blockRoot,
+                            finalizedCheckpoint.getEpochStartSlot(spec))
+                        .map(ancestorRoot -> ancestorRoot.equals(finalizedCheckpoint.getRoot()))
+                        .orElse(false);
+                  }
+                })
+        .orElseGet(() -> new GossipValidationHelper(spec, recentChainData, metricsSystem));
+  }
+
   @SuppressWarnings("unused")
   @JsonIgnoreProperties(ignoreUnknown = true)
   private static class GossipBeaconAggregateAndProofMetaData {
@@ -238,7 +265,7 @@ private static class GossipBeaconAggregateAndProofMetaData {
     private int blsSetting;
 
     @JsonProperty(value = "finalized_checkpoint", required = false)
-    private Object finalizedCheckpoint;
+    private FinalizedCheckpoint finalizedCheckpoint;
 
     public List<BlockEntry> getBlocks() {
       return blocks;
@@ -256,7 +283,7 @@ public BlsSetting getBlsSetting() {
       return BlsSetting.forCode(blsSetting);
     }
 
-    public Object getFinalizedCheckpoint() {
+    public FinalizedCheckpoint getFinalizedCheckpoint() {
       return finalizedCheckpoint;
     }
 
@@ -305,6 +332,38 @@ public String getExpected() {
         return expected;
       }
     }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private static class FinalizedCheckpoint {
+
+      @JsonProperty(value = "epoch", required = true)
+      private long epoch;
+
+      @JsonProperty(value = "root")
+      private String root;
+
+      @JsonProperty(value = "block")
+      private String block;
+
+      public String getBlock() {
+        return block;
+      }
+
+      public Checkpoint toCheckpoint(final TestDefinition testDefinition, final Spec spec) {
+        final Bytes32 checkpointRoot;
+        if (root != null) {
+          checkpointRoot = Bytes32.fromHexString(root);
+        } else if (block != null) {
+          final SignedBeaconBlock signedBlock =
+              loadSsz(testDefinition, block + ".ssz_snappy", spec::deserializeSignedBeaconBlock);
+          checkpointRoot = signedBlock.getRoot();
+        } else {
+          throw new IllegalStateException(
+              "finalized_checkpoint must specify either 'root' or 'block'");
+        }
+        return new Checkpoint(UInt64.valueOf(epoch), checkpointRoot);
+      }
+    }
   }
 
   private record BlockEntryAndBlock(