diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java index 68bf8cc8534b6..dd306a4d401d6 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Setting.parseInt; @@ -46,24 +45,44 @@ public class RecoverySettings { private static final Logger logger = LogManager.getLogger(RecoverySettings.class); + /** + * Undocumented setting, used to override the total physical available memory in tests + **/ + // package private for tests + static final Setting TOTAL_PHYSICAL_MEMORY_OVERRIDING_TEST_SETTING = Setting.byteSizeSetting( + "recovery_settings.total_physical_memory_override", + settings -> new ByteSizeValue(OsProbe.getInstance().getTotalPhysicalMemorySize()).getStringRep(), + Property.NodeScope + ); + + /** + * Undocumented setting, used to override the current JVM version in tests + **/ + // package private for tests + static final Setting JAVA_VERSION_OVERRIDING_TEST_SETTING = new Setting<>( + "recovery_settings.java_version_override", + settings -> JavaVersion.current().toString(), + JavaVersion::parse, + Property.NodeScope + ); + + public static final ByteSizeValue DEFAULT_MAX_BYTES_PER_SEC = new ByteSizeValue(40L, ByteSizeUnit.MB); + public static final Setting INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( "indices.recovery.max_bytes_per_sec", s -> { - final ByteSizeValue defaultMaxBytesPerSec = new ByteSizeValue(40, ByteSizeUnit.MB); final List roles = NodeRoleSettings.NODE_ROLES_SETTING.get(s); - final List dataRoles = roles.stream() - .filter(DiscoveryNodeRole::canContainData) - .collect(Collectors.toUnmodifiableList()); + final List dataRoles = roles.stream().filter(DiscoveryNodeRole::canContainData).toList(); if (dataRoles.isEmpty()) { // if the node is not a data node, this value doesn't matter, use the default - return defaultMaxBytesPerSec.getStringRep(); + return DEFAULT_MAX_BYTES_PER_SEC.getStringRep(); } if (dataRoles.stream() .allMatch( dn -> dn.equals(DiscoveryNodeRole.DATA_COLD_NODE_ROLE) || dn.equals(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) ) == false) { // the node is not a dedicated cold and/or frozen node, use the default - return defaultMaxBytesPerSec.getStringRep(); + return DEFAULT_MAX_BYTES_PER_SEC.getStringRep(); } /* * Now we are looking at a node that has a single data role, that data role is the cold data role, and the node does not @@ -71,11 +90,12 @@ public class RecoverySettings { * an assumption here that the size of the instance is correlated with I/O resources. That is we are assuming that the * larger the instance, the more disk and networking capacity it has available. */ - if (JavaVersion.current().compareTo(JavaVersion.parse("14")) < 0) { + final JavaVersion javaVersion = JAVA_VERSION_OVERRIDING_TEST_SETTING.get(s); + if (javaVersion.compareTo(JavaVersion.parse("14")) < 0) { // prior to JDK 14, the JDK did not take into consideration container memory limits when reporting total system memory - return defaultMaxBytesPerSec.getStringRep(); + return DEFAULT_MAX_BYTES_PER_SEC.getStringRep(); } - final ByteSizeValue totalPhysicalMemory = new ByteSizeValue(OsProbe.getInstance().getTotalPhysicalMemorySize()); + final ByteSizeValue totalPhysicalMemory = TOTAL_PHYSICAL_MEMORY_OVERRIDING_TEST_SETTING.get(s); final ByteSizeValue maxBytesPerSec; if (totalPhysicalMemory.compareTo(new ByteSizeValue(4, ByteSizeUnit.GB)) <= 0) { maxBytesPerSec = new ByteSizeValue(40, ByteSizeUnit.MB); @@ -375,6 +395,10 @@ private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) { } } + ByteSizeValue getMaxBytesPerSec() { + return maxBytesPerSec; + } + public int getMaxConcurrentFileChunks() { return maxConcurrentFileChunks; } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsTests.java index 28dfcdbc2055e..6ba323a48769a 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySettingsTests.java @@ -10,13 +10,28 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; +import org.elasticsearch.jdk.JavaVersion; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.indices.recovery.RecoverySettings.DEFAULT_MAX_BYTES_PER_SEC; +import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_SNAPSHOT_FILE_DOWNLOADS_PER_NODE; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_USE_SNAPSHOTS_SETTING; +import static org.elasticsearch.indices.recovery.RecoverySettings.JAVA_VERSION_OVERRIDING_TEST_SETTING; +import static org.elasticsearch.indices.recovery.RecoverySettings.TOTAL_PHYSICAL_MEMORY_OVERRIDING_TEST_SETTING; +import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -89,4 +104,201 @@ public void testMaxConcurrentSnapshotFileDownloadsPerNodeIsValidated() { ) ); } + + public void testDefaultMaxBytesPerSecOnNonDataNode() { + assertThat( + "Non-data nodes have a default 40mb rate limit", + nodeRecoverySettings().withRole(randomFrom("master", "ingest", "ml")).withRandomMemory().build().getMaxBytesPerSec(), + equalTo(DEFAULT_MAX_BYTES_PER_SEC) + ); + } + + public void testMaxBytesPerSecOnNonDataNodeWithIndicesRecoveryMaxBytesPerSec() { + final ByteSizeValue random = randomByteSizeValue(); + assertThat( + "Non-data nodes should use the defined rate limit when set", + nodeRecoverySettings().withRole(randomFrom("master", "ingest", "ml")) + .withIndicesRecoveryMaxBytesPerSec(random) + .withRandomMemory() + .build() + .getMaxBytesPerSec(), + equalTo(random) + ); + } + + public void testDefaultMaxBytesPerSecOnDataNode() { + assertThat( + "Data nodes that are not dedicated to cold/frozen have a default 40mb rate limit", + nodeRecoverySettings().withRole(randomFrom("data", "data_hot", "data_warm", "data_content")) + .withRandomMemory() + .build() + .getMaxBytesPerSec(), + equalTo(DEFAULT_MAX_BYTES_PER_SEC) + ); + } + + public void testMaxBytesPerSecOnDataNodeWithIndicesRecoveryMaxBytesPerSec() { + final Set roles = new HashSet<>(randomSubsetOf(randomIntBetween(1, 4), "data", "data_hot", "data_warm", "data_content")); + roles.addAll(randomSubsetOf(Set.of("data_cold", "data_frozen"))); + final ByteSizeValue random = randomByteSizeValue(); + assertThat( + "Data nodes that are not dedicated to cold/frozen should use the defined rate limit when set", + nodeRecoverySettings().withRoles(roles) + .withIndicesRecoveryMaxBytesPerSec(random) + .withRandomMemory() + .build() + .getMaxBytesPerSec(), + equalTo(random) + ); + } + + public void testDefaultMaxBytesPerSecOnColdOrFrozenNodeWithOldJvm() { + assertThat( + "Data nodes with only cold/frozen data roles have a default 40mb rate limit on Java version prior to 14", + nodeRecoverySettings().withRoles(randomFrom(Set.of("data_cold"), Set.of("data_frozen"), Set.of("data_cold", "data_frozen"))) + .withJavaVersion(randomFrom("8", "9", "11")) + .withRandomMemory() + .build() + .getMaxBytesPerSec(), + equalTo(DEFAULT_MAX_BYTES_PER_SEC) + ); + } + + public void testDefaultMaxBytesPerSecOnColdOrFrozenNode() { + final Set dataRoles = randomFrom(Set.of("data_cold"), Set.of("data_frozen"), Set.of("data_cold", "data_frozen")); + final String recentVersion = JavaVersion.current().compareTo(JavaVersion.parse("14")) < 0 ? "14" : null; + { + assertThat( + "Dedicated cold/frozen data nodes with <= 4GB of RAM have a default 40mb rate limit", + nodeRecoverySettings().withRoles(dataRoles) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(1L, ByteSizeUnit.GB.toBytes(4L)))) + .withJavaVersion(recentVersion) + .build() + .getMaxBytesPerSec(), + equalTo(new ByteSizeValue(40, ByteSizeUnit.MB)) + ); + } + { + assertThat( + "Dedicated cold/frozen data nodes with 4GB < RAM <= 8GB have a default 60mb rate limit", + nodeRecoverySettings().withRoles(dataRoles) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(ByteSizeUnit.GB.toBytes(4L) + 1L, ByteSizeUnit.GB.toBytes(8L)))) + .withJavaVersion(recentVersion) + .build() + .getMaxBytesPerSec(), + equalTo(new ByteSizeValue(60, ByteSizeUnit.MB)) + ); + } + { + assertThat( + "Dedicated cold/frozen data nodes with 8GB < RAM <= 16GB have a default 90mb rate limit", + nodeRecoverySettings().withRoles(dataRoles) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(ByteSizeUnit.GB.toBytes(8L) + 1L, ByteSizeUnit.GB.toBytes(16L)))) + .withJavaVersion(recentVersion) + .build() + .getMaxBytesPerSec(), + equalTo(new ByteSizeValue(90, ByteSizeUnit.MB)) + ); + } + { + assertThat( + "Dedicated cold/frozen data nodes with 16GB < RAM <= 32GB have a default 90mb rate limit", + nodeRecoverySettings().withRoles(dataRoles) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(ByteSizeUnit.GB.toBytes(16L) + 1L, ByteSizeUnit.GB.toBytes(32L)))) + .withJavaVersion(recentVersion) + .build() + .getMaxBytesPerSec(), + equalTo(new ByteSizeValue(125, ByteSizeUnit.MB)) + ); + } + { + assertThat( + "Dedicated cold/frozen data nodes with RAM > 32GB have a default 250mb rate limit", + nodeRecoverySettings().withRoles(dataRoles) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(ByteSizeUnit.GB.toBytes(32L) + 1L, ByteSizeUnit.TB.toBytes(4L)))) + .withJavaVersion(recentVersion) + .build() + .getMaxBytesPerSec(), + equalTo(new ByteSizeValue(250, ByteSizeUnit.MB)) + ); + } + } + + public void testMaxBytesPerSecOnColdOrFrozenNodeWithIndicesRecoveryMaxBytesPerSec() { + final ByteSizeValue random = randomByteSizeValue(); + assertThat( + "Dedicated cold/frozen data nodes should use the defined rate limit when set", + nodeRecoverySettings().withRoles(randomFrom(Set.of("data_cold"), Set.of("data_frozen"), Set.of("data_cold", "data_frozen"))) + .withJavaVersion(JavaVersion.current().compareTo(JavaVersion.parse("14")) < 0 ? "14" : null) + .withMemory(ByteSizeValue.ofBytes(randomLongBetween(1L, ByteSizeUnit.TB.toBytes(4L)))) + .withIndicesRecoveryMaxBytesPerSec(random) + .build() + .getMaxBytesPerSec(), + equalTo(random) + ); + } + + public static ByteSizeValue randomByteSizeValue() { + return new ByteSizeValue(randomLongBetween(0L, Long.MAX_VALUE >> 16)); + } + + public static ByteSizeValue randomNonZeroByteSizeValue() { + return new ByteSizeValue(randomLongBetween(1L, Long.MAX_VALUE >> 16)); + } + + static NodeRecoverySettings nodeRecoverySettings() { + return new NodeRecoverySettings(); + } + + private static class NodeRecoverySettings { + + private Set roles; + private ByteSizeValue physicalMemory; + private @Nullable String javaVersion; + private @Nullable ByteSizeValue indicesRecoveryMaxBytesPerSec; + + NodeRecoverySettings withRole(String role) { + this.roles = Set.of(Objects.requireNonNull(role)); + return this; + } + + NodeRecoverySettings withRoles(Set roles) { + this.roles = Objects.requireNonNull(roles); + return this; + } + + NodeRecoverySettings withMemory(ByteSizeValue physicalMemory) { + this.physicalMemory = Objects.requireNonNull(physicalMemory); + return this; + } + + NodeRecoverySettings withRandomMemory() { + return withMemory(ByteSizeValue.ofBytes(randomLongBetween(ByteSizeUnit.GB.toBytes(1L), ByteSizeUnit.TB.toBytes(4L)))); + } + + NodeRecoverySettings withJavaVersion(String javaVersion) { + this.javaVersion = javaVersion; + return this; + } + + NodeRecoverySettings withIndicesRecoveryMaxBytesPerSec(ByteSizeValue indicesRecoveryMaxBytesPerSec) { + this.indicesRecoveryMaxBytesPerSec = Objects.requireNonNull(indicesRecoveryMaxBytesPerSec); + return this; + } + + RecoverySettings build() { + final Settings.Builder settings = Settings.builder(); + settings.put(TOTAL_PHYSICAL_MEMORY_OVERRIDING_TEST_SETTING.getKey(), Objects.requireNonNull(physicalMemory)); + if (roles.isEmpty() == false) { + settings.putList(NODE_ROLES_SETTING.getKey(), new ArrayList<>(roles)); + } + if (javaVersion != null) { + settings.put(JAVA_VERSION_OVERRIDING_TEST_SETTING.getKey(), javaVersion); + } + if (indicesRecoveryMaxBytesPerSec != null) { + settings.put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), indicesRecoveryMaxBytesPerSec); + } + return new RecoverySettings(settings.build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + } + } }