diff --git a/ethereum/spec/build.gradle b/ethereum/spec/build.gradle index f5759305468..8c53b631026 100644 --- a/ethereum/spec/build.gradle +++ b/ethereum/spec/build.gradle @@ -8,6 +8,7 @@ dependencies { api project(':infrastructure:kzg') implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.yaml:snakeyaml' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation 'io.consensys.tuweni:tuweni-bytes' implementation 'io.consensys.tuweni:tuweni-ssz' diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigLoader.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigLoader.java index 2ad83a6ce88..8ff729fe49c 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigLoader.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigLoader.java @@ -60,17 +60,17 @@ public static SpecConfigAndParent loadConfig( } public static SpecConfigAndParent loadRemoteConfig( - final Map config) { + final Map config) { final SpecConfigReader reader = new SpecConfigReader(); if (config.containsKey(SpecConfigReader.PRESET_KEY)) { try { - applyPreset("remote", reader, true, config.get(SpecConfigReader.PRESET_KEY)); + applyPreset("remote", reader, true, (String) config.get(SpecConfigReader.PRESET_KEY)); } catch (IOException e) { throw new UncheckedIOException(e); } } if (config.containsKey(SpecConfigReader.CONFIG_NAME_KEY)) { - final String configNameKey = config.get(SpecConfigReader.CONFIG_NAME_KEY); + final String configNameKey = (String) config.get(SpecConfigReader.CONFIG_NAME_KEY); try { processConfig(configNameKey, reader, true); } catch (IllegalArgumentException exception) { @@ -87,9 +87,9 @@ public static SpecConfigAndParent loadRemoteConfig( static void processConfig( final String source, final SpecConfigReader reader, final boolean ignoreUnknownConfigItems) { try (final InputStream configFile = loadConfigurationFile(source)) { - final Map configValues = reader.readValues(configFile); + final Map configValues = reader.readValues(configFile); final Optional maybePreset = - Optional.ofNullable(configValues.get(SpecConfigReader.PRESET_KEY)); + Optional.ofNullable((String) configValues.get(SpecConfigReader.PRESET_KEY)); // Legacy config files won't have a preset field if (maybePreset.isPresent()) { diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigReader.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigReader.java index bfbe701b5db..b38d4568c15 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigReader.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigReader.java @@ -15,9 +15,7 @@ import static tech.pegasys.teku.spec.config.SpecConfigFormatter.camelToSnakeCase; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.RuntimeJsonMappingException; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -131,14 +129,14 @@ public SpecConfigAndParent build( */ public void readAndApply(final InputStream source, final boolean ignoreUnknownConfigItems) throws IOException { - final Map rawValues = readValues(source); + final Map rawValues = readValues(source); loadFromMap(rawValues, ignoreUnknownConfigItems); } public void loadFromMap( - final Map rawValues, final boolean ignoreUnknownConfigItems) { - final Map unprocessedConfig = new HashMap<>(rawValues); - final Map apiSpecConfig = new HashMap<>(rawValues); + final Map rawValues, final boolean ignoreUnknownConfigItems) { + final Map unprocessedConfig = new HashMap<>(rawValues); + final Map apiSpecConfig = new HashMap<>(rawValues); // Remove any keys that we're ignoring KEYS_TO_IGNORE.forEach( key -> { @@ -240,16 +238,10 @@ public void loadFromMap( } } - @SuppressWarnings("unchecked") - public Map readValues(final InputStream source) throws IOException { - final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + public Map readValues(final InputStream source) throws IOException { + final YamlConfigReader reader = new YamlConfigReader(); try { - return (Map) - mapper - .readerFor( - mapper.getTypeFactory().constructMapType(Map.class, String.class, String.class)) - .readValues(source) - .next(); + return reader.readValues(source); } catch (NoSuchElementException e) { throw new IllegalArgumentException("Supplied spec config is empty"); } catch (RuntimeJsonMappingException e) { diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/YamlConfigReader.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/YamlConfigReader.java new file mode 100644 index 00000000000..39fb4dee4b1 --- /dev/null +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/YamlConfigReader.java @@ -0,0 +1,61 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.config; + +import java.io.InputStream; +import java.util.Map; +import java.util.NoSuchElementException; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.resolver.Resolver; + +public class YamlConfigReader { + private final Yaml yaml; + + public YamlConfigReader() { + final Resolver resolver = new CustomResolver(); + yaml = + new Yaml( + new Constructor(new LoaderOptions()), + new Representer(new DumperOptions()), + new DumperOptions(), + resolver); + } + + public Map readValues(final InputStream source) { + final Map values = yaml.load(source); + if (values == null) { + throw new NoSuchElementException(); + } + return values; + } + + /* + * For the purposes of reading config, we just want strings, we don't want numeric values. + * This will ensure that we get hex values through as unparsed, and ensure we're getting + * things such as CRC checks through without them being altered. + * The implicit resolver below is basically saying just match numerics as string. + */ + public static class CustomResolver extends Resolver { + @Override + protected void addImplicitResolvers() { + addImplicitResolver(Tag.STR, INT, "+-0123456789."); + super.addImplicitResolvers(); + } + } +} diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigLoaderTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigLoaderTest.java index df29aefe455..219bf0aef05 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigLoaderTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigLoaderTest.java @@ -19,9 +19,7 @@ import static tech.pegasys.teku.spec.config.SpecConfigAssertions.assertAllBellatrixFieldsSet; import static tech.pegasys.teku.spec.config.SpecConfigAssertions.assertAllFieldsSet; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Resources; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; @@ -55,7 +53,7 @@ public void shouldLoadAllKnownNetworks(final Eth2Network network) throws Excepti * sufficient. */ @ParameterizedTest(name = "{0}") - @ValueSource(strings = {"holesky", "mainnet"}) + @ValueSource(strings = {"hoodi", "mainnet"}) public void shouldMaintainConfigNameBackwardsCompatibility(final String name) { final SpecConfig config = SpecConfigLoader.loadConfig(name).specConfig(); assertThat(config.getRawConfig().get("CONFIG_NAME")).isEqualTo(name); @@ -161,10 +159,8 @@ private InputStream loadInvalidFile(final String file) { return getClass().getResourceAsStream("invalid/" + file); } - private Map readJsonConfig(final InputStream source) throws IOException { - final ObjectMapper mapper = new ObjectMapper(); - return mapper - .readerFor(mapper.getTypeFactory().constructMapType(Map.class, String.class, String.class)) - .readValue(source); + private Map readJsonConfig(final InputStream source) { + YamlConfigReader reader = new YamlConfigReader(); + return reader.readValues(source); } } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigReaderTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigReaderTest.java index a923eb7c491..f6b2f0363f5 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigReaderTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/SpecConfigReaderTest.java @@ -13,14 +13,19 @@ package tech.pegasys.teku.spec.config; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static tech.pegasys.teku.spec.config.SpecConfigAssertions.assertAllAltairFieldsSet; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import tech.pegasys.teku.infrastructure.unsigned.UInt64; public class SpecConfigReaderTest { @@ -141,7 +146,7 @@ public void read_invalidLong_wrongType() { assertThatThrownBy(() -> readConfig(stream)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( - "Cannot read spec config: Cannot deserialize value of type `java.lang.String` from Array")); + "Failed to parse value for constant VALIDATOR_REGISTRY_LIMIT")); } @Test @@ -187,6 +192,19 @@ public void read_invalidUInt64_tooLarge() { "Failed to parse value for constant MIN_GENESIS_TIME: '18446744073709552001'")); } + @Test + public void read_localConfigFile_notLoadingDefaults(@TempDir final Path tempDir) + throws IOException { + Files.writeString( + tempDir.resolve("test.yaml"), "PRESET_BASE: 'mainnet'\nCONFIG_NAME: 'mainnet'", UTF_8); + Map data = + reader.readValues(Files.newInputStream(tempDir.resolve("test.yaml"))); + reader.loadFromMap(data, true); + assertThatThrownBy(reader::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("SECONDS_PER_ETH1_BLOCK"); + } + @Test public void read_invalidBytes4_tooLarge() { processFileAsInputStream( diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/YamlConfigReaderTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/YamlConfigReaderTest.java new file mode 100644 index 00000000000..cdb74dfee04 --- /dev/null +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/config/YamlConfigReaderTest.java @@ -0,0 +1,73 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.config; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +public class YamlConfigReaderTest { + private final YamlConfigReader reader = new YamlConfigReader(); + + @Test + void shouldReadSimpleObjectToStringMap() { + final String testData = + """ +version: 250 +info: the info +"""; + final Map objData = reader.readValues(IOUtils.toInputStream(testData, UTF_8)); + assertThat(objData.get("version")).isInstanceOf(String.class).isEqualTo("250"); + assertThat(objData.get("info")).isInstanceOf(String.class).isEqualTo("the info"); + } + + @Test + void shouldReadObjectWithListOfObjects() { + final String testData = + """ +data: + - a: 1 + aa: 11 + - b: two + bb: three +"""; + + final Map objData = reader.readValues(IOUtils.toInputStream(testData, UTF_8)); + assertThat(objData.get("data")).isInstanceOf(List.class); + final List> list = (List>) objData.get("data"); + for (Object o : list) { + assertThat(o).isInstanceOf(Map.class); + assertThat(((Map) o).size()).isEqualTo(2); + } + assertThat(list) + .containsExactly(Map.of("a", "1", "aa", "11"), Map.of("b", "two", "bb", "three")); + } + + @Test + void shouldReadEth1Address() { + final String testData = + """ +DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 + """; + final Map objData = reader.readValues(IOUtils.toInputStream(testData, UTF_8)); + assertThat(objData.get("DEPOSIT_CONTRACT_ADDRESS")) + .isInstanceOf(String.class) + .isEqualTo("0x4242424242424242424242424242424242424242"); + } +} diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 0b385aa961c..abcfa46bca0 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -16,6 +16,8 @@ dependencyManagement { dependency 'com.google.guava:guava:33.1.0-jre' + dependency 'org.yaml:snakeyaml:2.4' + dependency 'org.jsoup:jsoup:1.20.1' dependency 'com.launchdarkly:okhttp-eventsource:4.1.1' diff --git a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoader.java b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoader.java index 8d20a868b2f..8202797486a 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoader.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoader.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.cli.subcommand; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -47,7 +48,7 @@ static Spec getSpec(final OkHttpValidatorMinimalTypeDefClient apiClient) { try { return apiClient .getSpec() - .map(SpecConfigLoader::loadRemoteConfig) + .map(config -> SpecConfigLoader.loadRemoteConfig(new HashMap<>(config))) .map(SpecFactory::create) .orElseThrow(); } catch (final Throwable ex) { @@ -78,13 +79,13 @@ private static Spec getSpecWithFailovers( throw new InvalidConfigurationException(errMsg); } - private static T retry(final Callable f) { + private static T retry(final Callable callable) { try { - return f.call(); + return callable.call(); } catch (Throwable ex) { logError(ex); sleep(); - return retry(f); + return retry(callable); } } diff --git a/teku/src/test/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoaderTest.java b/teku/src/test/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoaderTest.java index 9c39079fbfc..0acbda73ebd 100644 --- a/teku/src/test/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoaderTest.java +++ b/teku/src/test/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoaderTest.java @@ -22,6 +22,7 @@ import com.google.common.io.Resources; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Test; @@ -70,7 +71,8 @@ void shouldDefaultNetworkConfigThatMovedFromConstants() throws IOException { final ObjectMapper objectMapper = new ObjectMapper(); TypeReference> typeReference = new TypeReference<>() {}; Map data = objectMapper.readValue(jsonConfig, typeReference); - final SpecConfig specConfig = SpecConfigLoader.loadRemoteConfig(data).specConfig(); + final SpecConfig specConfig = + SpecConfigLoader.loadRemoteConfig(new HashMap<>(data)).specConfig(); // Check values not assigned, using default values assertThat(specConfig.getMaxPayloadSize()).isEqualTo(10485760);