Peerdas + master upstream + test fixes

.gitignore

@@ -4,7 +4,6 @@
 *~.nib
 *.iml
 *.launch
-*.swp
 *.log
 .lh/*
 /db/

CHANGELOG.md

@@ -7,10 +7,15 @@
 ## Unreleased Changes
 
 ### Breaking Changes
+- Previous versions of teku will not be able to read configuration from this version of teku (including VC) due to BPO configuration being added.
 
 ### Additions and Improvements
 - Added `--p2p-static-peers-url` option to read static peers from a URL or file
 - Added node epoch and computed slot to the sync committee duties failure message for more context about the failure condition.
 - Updated third party libraries.
+- Added an info message on startup for the highest supported milestone and associated epoch.
+- Added jdk 24 docker image build.
+- Improved performance when scheduling attestations in the beginning of the epoch for a large number of validators.
+- Improved configuration loading to use builtin configurations to default any fields we need that were missing from a passed in configuration.
 
 ### Bug Fixes

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/das/DasSyncAcceptanceTest.java

@@ -15,6 +15,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import com.google.common.io.Resources;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Optional;
@@ -98,7 +99,7 @@ private int getAndAssertDasCustody(final TekuBeaconNode node, final UInt64 fuluS
   }
 
   private TekuBeaconNode createLateJoiningNode(
-      final TekuBeaconNode primaryNode, final int genesisTime) throws IOException {
+      final TekuBeaconNode primaryNode, final int genesisTime) throws Exception {
     return createTekuBeaconNode(
         createConfigBuilder()
             .withGenesisTime(genesisTime)
@@ -108,17 +109,10 @@ private TekuBeaconNode createLateJoiningNode(
             .build());
   }
 
-  private TekuNodeConfigBuilder createConfigBuilder() throws IOException {
+  private TekuNodeConfigBuilder createConfigBuilder() throws Exception {
     return TekuNodeConfigBuilder.createBeaconNode()
-        .withNetwork("minimal")
-        .withAltairEpoch(UInt64.valueOf(0))
-        .withBellatrixEpoch(UInt64.valueOf(0))
-        .withCapellaEpoch(UInt64.valueOf(0))
-        .withDenebEpoch(UInt64.valueOf(0))
-        .withElectraEpoch(UInt64.valueOf(0))
-        .withFuluEpoch(UInt64.valueOf(fuluEpoch))
+        .withNetwork(Resources.getResource("fulu-minimal.yaml"))
         .withStubExecutionEngine()
-        .withStubBlobCount(Optional.of(1))
         .withLogLevel("DEBUG");
   }
 }

acceptance-tests/src/acceptance-test/resources/fulu-minimal.yaml

@@ -0,0 +1,175 @@
+# Minimal config
+
+# Extends the minimal preset
+PRESET_BASE: 'minimal'
+
+# Free-form short name of the network that this configuration applies to - known
+# canonical network names include:
+# * 'mainnet' - there can be only one
+# * 'prater' - testnet
+# Must match the regex: [a-z0-9\-]
+CONFIG_NAME: 'minimal'
+
+# Transition
+# ---------------------------------------------------------------
+# 2**256-2**10 for testing minimal network
+TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912
+# By default, don't use these params
+TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
+TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
+
+
+
+# Genesis
+# ---------------------------------------------------------------
+# [customized]
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64
+# Jan 3, 2020
+MIN_GENESIS_TIME: 1578009600
+# Highest byte set to 0x01 to avoid collisions with mainnet versioning
+GENESIS_FORK_VERSION: 0x00000001
+# [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis
+GENESIS_DELAY: 300
+
+
+# Forking
+# ---------------------------------------------------------------
+# Values provided for illustrative purposes.
+# Individual tests/testnets may set different values.
+
+# Altair
+ALTAIR_FORK_VERSION: 0x01000001
+ALTAIR_FORK_EPOCH: 0
+# Bellatrix
+BELLATRIX_FORK_VERSION: 0x02000001
+BELLATRIX_FORK_EPOCH: 0
+# Capella
+CAPELLA_FORK_VERSION: 0x03000001
+CAPELLA_FORK_EPOCH: 0
+# Deneb
+DENEB_FORK_VERSION: 0x04000001
+DENEB_FORK_EPOCH: 0
+# Electra
+ELECTRA_FORK_VERSION: 0x05000001
+ELECTRA_FORK_EPOCH: 0
+# Fulu
+FULU_FORK_VERSION: 0x06000001
+FULU_FORK_EPOCH: 1
+
+# Time parameters
+# ---------------------------------------------------------------
+# [customized] Faster for testing purposes
+SECONDS_PER_SLOT: 6
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 14
+# 2**8 (= 256) epochs
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit
+SHARD_COMMITTEE_PERIOD: 64
+# [customized] process deposits more quickly, but insecure
+ETH1_FOLLOW_DISTANCE: 16
+
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# [customized] more easily demonstrate the difference between this value and the activation churn limit
+MIN_PER_EPOCH_CHURN_LIMIT: 2
+# [customized] scale queue churn at much lower validator counts for testing
+CHURN_LIMIT_QUOTIENT: 32
+# [New in Deneb:EIP7514] [customized]
+MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4
+
+
+# Fork choice
+# ---------------------------------------------------------------
+# 40%
+PROPOSER_SCORE_BOOST: 40
+# 20%
+REORG_HEAD_WEIGHT_THRESHOLD: 20
+# 160%
+REORG_PARENT_WEIGHT_THRESHOLD: 160
+# `2` epochs
+REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
+
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Ethereum Goerli testnet
+DEPOSIT_CHAIN_ID: 5
+DEPOSIT_NETWORK_ID: 5
+# Configured on a per testnet basis
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
+
+
+# Networking
+# ---------------------------------------------------------------
+# `10 * 2**20` (= 10485760, 10 MiB)
+MAX_PAYLOAD_SIZE: 10485760
+# `2**10` (= 1024)
+MAX_REQUEST_BLOCKS: 1024
+# `2**8` (= 256)
+EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
+# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
+MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
+# 5s
+TTFB_TIMEOUT: 5
+# 10s
+RESP_TIMEOUT: 10
+ATTESTATION_PROPAGATION_SLOT_RANGE: 32
+# 500ms
+MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500
+MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000
+MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
+# 2 subnets per node
+SUBNETS_PER_NODE: 2
+# 2**8 (= 64)
+ATTESTATION_SUBNET_COUNT: 64
+ATTESTATION_SUBNET_EXTRA_BITS: 0
+# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
+ATTESTATION_SUBNET_PREFIX_BITS: 6
+
+# Deneb
+# `2**7` (=128)
+MAX_REQUEST_BLOCKS_DENEB: 128
+# `2**12` (= 4096 epochs, ~18 days)
+MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
+# `6`
+BLOB_SIDECAR_SUBNET_COUNT: 6
+# `6`
+MAX_BLOBS_PER_BLOCK: 6
+# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
+MAX_REQUEST_BLOB_SIDECARS: 768
+
+# Electra
+# 2**6 * 10**9 (= 64,000,000,000)
+MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000
+# 2**7 * 10**9 (= 128,000,000,000)
+MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 128000000000
+# `9`
+BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9
+# `uint64(9)`
+MAX_BLOBS_PER_BLOCK_ELECTRA: 9
+# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA
+MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152
+
+# Fulu
+NUMBER_OF_COLUMNS: 128
+NUMBER_OF_CUSTODY_GROUPS: 128
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
+MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
+SAMPLES_PER_SLOT: 8
+CUSTODY_REQUIREMENT: 4
+VALIDATOR_CUSTODY_REQUIREMENT: 8
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
+BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
+
+BLOB_SCHEDULE:
+  # Deneb
+  - EPOCH: 0
+    MAX_BLOBS_PER_BLOCK: 9

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/Node.java

@@ -253,7 +253,11 @@ protected void copyDirectoryToTar(final String sourcePath, final File localTarge
   }
 
   protected static File copyToTmpFile(final URL fileUrl) throws Exception {
-    final File tmpFile = File.createTempFile("teku", ".tmp");
+    return copyToTmpFile(fileUrl, ".tmp");
+  }
+
+  protected static File copyToTmpFile(final URL fileUrl, final String suffix) throws Exception {
+    final File tmpFile = File.createTempFile("teku", suffix);
     tmpFile.deleteOnExit();
     try (InputStream inputStream = fileUrl.openStream();
         FileOutputStream out = new FileOutputStream(tmpFile)) {

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java

@@ -100,7 +100,18 @@ private TekuBeaconNode(
       final Network network, final TekuDockerVersion version, final TekuNodeConfig tekuNodeConfig) {
     super(network, TEKU_DOCKER_IMAGE_NAME, version, LOG);
     this.config = tekuNodeConfig;
-    this.spec = SpecFactory.create(config.getNetworkName(), config.getSpecConfigModifier());
+    if (config.getConfigFileMap().containsValue(NETWORK_FILE_PATH)) {
+      final String tmpConfigFilePath =
+          config.getConfigFileMap().entrySet().stream()
+              .filter(entry -> entry.getValue().equals(NETWORK_FILE_PATH))
+              .findFirst()
+              .orElseThrow()
+              .getKey()
+              .getAbsolutePath();
+      this.spec = SpecFactory.create(tmpConfigFilePath, config.getSpecConfigModifier());
+    } else {
+      this.spec = SpecFactory.create(config.getNetworkName(), config.getSpecConfigModifier());
+    }
     if (config.getConfigMap().containsKey("validator-api-enabled")) {
       container.addExposedPort(VALIDATOR_API_PORT);
     }
@@ -675,7 +686,7 @@ private Optional<Bytes32> fetchValidatorWithdrawalCredentials(final int validato
 
   public int getDataColumnSidecarCount(final String blockId) throws IOException {
     final String result =
-        httpClient.get(getRestApiUrl(), "/eth/v1/beacon/data_column_sidecars/" + blockId);
+        httpClient.get(getRestApiUrl(), "/teku/v1/beacon/data_column_sidecars/" + blockId);
     final JsonNode jsonNode = OBJECT_MAPPER.readTree(result);
     return jsonNode.get("data").size();
   }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNodeConfigBuilder.java

@@ -275,12 +275,10 @@ public TekuNodeConfigBuilder withNetwork(final String networkName) {
     return this;
   }
 
-  public TekuNodeConfigBuilder withNetwork(final URL networkYaml, final String networkName)
-      throws Exception {
-    LOG.debug("Network={}", networkName);
+  public TekuNodeConfigBuilder withNetwork(final URL networkYaml) throws Exception {
     LOG.debug("Copy Network from URL {}", networkYaml);
     configMap.put("network", NETWORK_FILE_PATH);
-    configFileMap.put(copyToTmpFile(networkYaml), NETWORK_FILE_PATH);
+    configFileMap.put(Node.copyToTmpFile(networkYaml, ".yaml"), NETWORK_FILE_PATH);
     return this;
   }
 
@@ -646,16 +644,6 @@ public TekuNodeConfigBuilder withDasExtraCustodyGroupCount(final int extraCustod
     return this;
   }
 
-  public TekuNodeConfigBuilder withStubBlobCount(final Optional<Integer> stubBlobCount) {
-    LOG.debug("Xinterop-number-of-blobs: {}", stubBlobCount);
-    if (stubBlobCount.isPresent()) {
-      configMap.put("Xinterop-number-of-blobs", stubBlobCount.get());
-    } else {
-      configMap.remove("Xinterop-number-of-blobs");
-    }
-    return this;
-  }
-
   private TekuNodeConfigBuilder withPrivateKey(final PrivKey privKey) throws IOException {
     mustBe(NodeType.BEACON_NODE, NodeType.BOOTNODE);
     this.maybePrivKey = Optional.ofNullable(privKey);

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java

@@ -51,7 +51,7 @@
 import tech.pegasys.teku.spec.datastructures.operations.Attestation;
 import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SyncCommitteeMessage;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
-import tech.pegasys.teku.statetransition.attestation.utils.AttestationBitsAggregator;
+import tech.pegasys.teku.statetransition.attestation.utils.AttestationBits;
 import tech.pegasys.teku.storage.client.CombinedChainDataClient;
 import tech.pegasys.teku.validator.api.ValidatorPerformanceTrackingMode;
 import tech.pegasys.teku.validator.coordinator.ActiveValidatorTracker;
@@ -304,19 +304,19 @@ private AttestationPerformance calculateAttestationPerformance(
 
     // Pre-process attestations included on chain to group them by
     // data hash to inclusion slot to aggregation bitlist
-    final Map<Bytes32, NavigableMap<UInt64, AttestationBitsAggregator>>
-        slotAndBitlistsByAttestationDataHash = new HashMap<>();
+    final Map<Bytes32, NavigableMap<UInt64, AttestationBits>> slotAndBitlistsByAttestationDataHash =
+        new HashMap<>();
     for (final Map.Entry<UInt64, List<Attestation>> entry :
         attestationsIncludedOnChain.entrySet()) {
       for (final Attestation attestation : entry.getValue()) {
         final Optional<Int2IntMap> committeesSize = getCommitteesSize(attestation, state);
         final Bytes32 attestationDataHash = attestation.getData().hashTreeRoot();
-        final NavigableMap<UInt64, AttestationBitsAggregator> slotToBitlists =
+        final NavigableMap<UInt64, AttestationBits> slotToBitlists =
             slotAndBitlistsByAttestationDataHash.computeIfAbsent(
                 attestationDataHash, __ -> new TreeMap<>());
         slotToBitlists.merge(
             entry.getKey(),
-            AttestationBitsAggregator.of(attestation, committeesSize),
+            AttestationBits.of(attestation, committeesSize),
             (firstBitsAggregator, secondBitsAggregator) -> {
               firstBitsAggregator.or(secondBitsAggregator);
               return firstBitsAggregator;
@@ -330,7 +330,7 @@ private AttestationPerformance calculateAttestationPerformance(
       if (!slotAndBitlistsByAttestationDataHash.containsKey(sentAttestationDataHash)) {
         continue;
       }
-      final NavigableMap<UInt64, AttestationBitsAggregator> slotAndBitlists =
+      final NavigableMap<UInt64, AttestationBits> slotAndBitlists =
           slotAndBitlistsByAttestationDataHash.get(sentAttestationDataHash);
       for (UInt64 slot : slotAndBitlists.keySet()) {
         if (slotAndBitlists.get(slot).isSuperSetOf(sentAttestation)) {

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTestDeneb.java

@@ -48,8 +48,8 @@
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBodySchema;
-import tech.pegasys.teku.spec.datastructures.blocks.versions.deneb.SignedBlockContentsDeneb;
 import tech.pegasys.teku.spec.datastructures.builder.BuilderPayload;
 import tech.pegasys.teku.spec.datastructures.builder.versions.deneb.BlobsBundleDeneb;
 import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle;
@@ -275,8 +275,7 @@ void shouldGetBlobsBundleForLocallyProducedBlocksViaFallback() {
 
   @Test
   void shouldCreateBlobSidecarsForBlockContents() {
-    final SignedBlockContentsDeneb signedBlockContents =
-        dataStructureUtil.randomSignedBlockContentsDeneb();
+    final SignedBlockContainer signedBlockContents = dataStructureUtil.randomSignedBlockContents();
 
     final MiscHelpersDeneb miscHelpersDeneb =
         MiscHelpersDeneb.required(spec.atSlot(signedBlockContents.getSlot()).miscHelpers());

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTestFulu.java

@@ -50,8 +50,8 @@
 import tech.pegasys.teku.spec.datastructures.blobs.versions.fulu.DataColumnSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBodySchema;
-import tech.pegasys.teku.spec.datastructures.blocks.versions.fulu.SignedBlockContentsFulu;
 import tech.pegasys.teku.spec.datastructures.builder.BuilderPayload;
 import tech.pegasys.teku.spec.datastructures.builder.versions.fulu.BlobsBundleFulu;
 import tech.pegasys.teku.spec.datastructures.execution.BlobsCellBundle;
@@ -285,8 +285,7 @@ void shouldGetBlobsCellBundleForLocallyProducedBlocksViaFallback() {
 
   @Test
   void shouldCreateDataColumnSidecarsForBlockContents() {
-    final SignedBlockContentsFulu signedBlockContents =
-        dataStructureUtil.randomSignedBlockContentsFulu();
+    final SignedBlockContainer signedBlockContents = dataStructureUtil.randomSignedBlockContents();
 
     final MiscHelpersFulu miscHelpersFulu =
         MiscHelpersFulu.required(spec.atSlot(signedBlockContents.getSlot()).miscHelpers());