From 2d6002125aefdf803c1c2b56e0c4f6b565f19b55 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Mon, 2 Jun 2025 13:46:49 +1000 Subject: [PATCH 01/30] Add nodeHeapUsage field to ClusterInfo --- .../org/elasticsearch/TransportVersions.java | 1 + .../elasticsearch/cluster/ClusterInfo.java | 15 ++++- .../cluster/ClusterInfoSimulator.java | 1 + .../org/elasticsearch/cluster/HeapUsage.java | 65 +++++++++++++++++++ .../cluster/InternalClusterInfoService.java | 5 +- .../cluster/ClusterInfoTests.java | 20 +++++- .../ExpectedShardSizeEstimatorTests.java | 1 + .../AllocationStatsServiceTests.java | 1 + .../allocation/DiskThresholdMonitorTests.java | 2 +- .../ExpectedShardSizeAllocationTests.java | 3 +- .../BalancedShardsAllocatorTests.java | 3 +- .../ClusterAllocationSimulationTests.java | 2 +- .../allocator/ClusterBalanceStatsTests.java | 1 + .../allocator/ClusterInfoSimulatorTests.java | 10 ++- .../DesiredBalanceComputerTests.java | 4 +- .../DesiredBalanceReconcilerTests.java | 1 + .../decider/DiskThresholdDeciderTests.java | 2 +- .../DiskThresholdDeciderUnitTests.java | 14 +++- .../ReactiveStorageDeciderService.java | 1 + ...oscalingCalculateCapacityServiceTests.java | 4 +- .../FrozenStorageDeciderServiceTests.java | 2 +- .../ProactiveStorageDeciderServiceTests.java | 2 +- .../ReactiveStorageDeciderServiceTests.java | 4 +- ...nsportNodeDeprecationCheckActionTests.java | 1 + 24 files changed, 146 insertions(+), 19 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/HeapUsage.java diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 57ba6de0b973c..94c46fd830bdc 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -273,6 +273,7 @@ static TransportVersion def(int id) { public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00); public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00); public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00); + public static final TransportVersion HEAP_USAGE_IN_CLUSTER_INFO = def(9_087_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index a2c260e8699e8..be1d8ff51694a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -57,9 +57,10 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { final Map shardDataSetSizes; final Map dataPath; final Map reservedSpace; + final Map nodeHeapUsage; protected ClusterInfo() { - this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); + this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); } /** @@ -79,7 +80,8 @@ public ClusterInfo( Map shardSizes, Map shardDataSetSizes, Map dataPath, - Map reservedSpace + Map reservedSpace, + Map nodeHeapUsage ) { this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage); this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage); @@ -87,6 +89,7 @@ public ClusterInfo( this.shardDataSetSizes = Map.copyOf(shardDataSetSizes); this.dataPath = Map.copyOf(dataPath); this.reservedSpace = Map.copyOf(reservedSpace); + this.nodeHeapUsage = Map.copyOf(nodeHeapUsage); } public ClusterInfo(StreamInput in) throws IOException { @@ -98,6 +101,11 @@ public ClusterInfo(StreamInput in) throws IOException { ? in.readImmutableMap(NodeAndShard::new, StreamInput::readString) : in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString); this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new); + if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { + this.nodeHeapUsage = in.readImmutableMap(HeapUsage::new); + } else { + this.nodeHeapUsage = Map.of(); + } } @Override @@ -112,6 +120,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(this.dataPath, (o, k) -> createFakeShardRoutingFromNodeAndShard(k).writeTo(o), StreamOutput::writeString); } out.writeMap(this.reservedSpace); + if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { + out.writeMap(this.nodeHeapUsage, StreamOutput::writeWriteable); + } } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java index 17cf5c7b8b7c7..0536322b1d730 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java @@ -153,6 +153,7 @@ public ClusterInfo getClusterInfo() { shardSizes.toImmutableMap(), shardDataSetSizes, dataPath, + Map.of(), Map.of() ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java new file mode 100644 index 0000000000000..d28c80cd3bf54 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Record representing the heap usage for a single cluster node + */ +public record HeapUsage(String nodeId, String nodeName, long totalBytes, long freeBytes) implements ToXContentFragment, Writeable { + + public HeapUsage(StreamInput in) throws IOException { + this(in.readString(), in.readString(), in.readVLong(), in.readVLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.nodeId); + out.writeString(this.nodeName); + out.writeVLong(this.totalBytes); + out.writeVLong(this.freeBytes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("node_id", this.nodeId); + builder.field("node_name", this.nodeName); + builder.humanReadableField("total_heap_bytes", "total", ByteSizeValue.ofBytes(this.totalBytes)); + builder.humanReadableField("used_heap_bytes", "used", ByteSizeValue.ofBytes(this.usedBytes())); + builder.humanReadableField("free_heap_bytes", "free", ByteSizeValue.ofBytes(this.freeBytes)); + builder.field("free_heap_percent", truncatePercent(this.freeHeapAsPercentage())); + builder.field("used_heap_percent", truncatePercent(this.usedHeapAsPercentage())); + return builder; + } + + public double freeHeapAsPercentage() { + return 100.0 * freeBytes / (double) totalBytes; + } + + public double usedHeapAsPercentage() { + return 100.0 - freeHeapAsPercentage(); + } + + public long usedBytes() { + return totalBytes - freeBytes; + } + + private static double truncatePercent(double pct) { + return Math.round(pct * 10.0) / 10.0; + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 17bd913ffb724..29d699ec26d48 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -89,6 +89,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; private volatile IndicesStatsSummary indicesStatsSummary; + private volatile Map nodeHeapUsages; private final ThreadPool threadPool; private final Client client; @@ -104,6 +105,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); + this.nodeHeapUsages = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; @@ -413,7 +415,8 @@ public ClusterInfo getClusterInfo() { indicesStatsSummary.shardSizes, indicesStatsSummary.shardDataSetSizes, indicesStatsSummary.dataPath, - indicesStatsSummary.reservedSpace + indicesStatsSummary.reservedSpace, + nodeHeapUsages ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java index 70adfa61dd853..28cbd89294c30 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java @@ -41,10 +41,28 @@ public static ClusterInfo randomClusterInfo() { randomShardSizes(), randomDataSetSizes(), randomRoutingToDataPath(), - randomReservedSpace() + randomReservedSpace(), + randomNodeHeapUsage() ); } + private static Map randomNodeHeapUsage() { + int numEntries = randomIntBetween(0, 128); + Map nodeHeapUsage = new HashMap<>(numEntries); + for (int i = 0; i < numEntries; i++) { + String key = randomAlphaOfLength(32); + final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE); + final HeapUsage diskUsage = new HeapUsage( + randomAlphaOfLength(4), + randomAlphaOfLength(4), + totalBytes, + randomIntBetween(0, totalBytes) + ); + nodeHeapUsage.put(key, diskUsage); + } + return nodeHeapUsage; + } + private static Map randomDiskUsage() { int numEntries = randomIntBetween(0, 128); Map builder = new HashMap<>(numEntries); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java index a2e2f3326f527..754b4d2b22d0d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java @@ -205,6 +205,7 @@ private static ClusterInfo createClusterInfo(ShardRouting shard, Long size) { Map.of(ClusterInfo.shardIdentifierFromRouting(shard), size), Map.of(), Map.of(), + Map.of(), Map.of() ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java index 4a07b837b08af..14e0aaa253749 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java @@ -78,6 +78,7 @@ public void testShardStats() { Map.of(ClusterInfo.shardIdentifierFromRouting(shardId, true), currentShardSize), Map.of(), Map.of(), + Map.of(), Map.of() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 6ce417456d30a..df0fa875a7249 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -1580,7 +1580,7 @@ private static ClusterInfo clusterInfo( Map diskUsages, Map reservedSpace ) { - return new ClusterInfo(diskUsages, Map.of(), Map.of(), Map.of(), Map.of(), reservedSpace); + return new ClusterInfo(diskUsages, Map.of(), Map.of(), Map.of(), Map.of(), reservedSpace, Map.of()); } private static DiscoveryNode newFrozenOnlyNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java index cea6b24684978..f1a2b4b1358fe 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java @@ -258,11 +258,12 @@ private static ClusterInfo createClusterInfoWith(ShardId shardId, long size) { ), Map.of(), Map.of(), + Map.of(), Map.of() ); } private static ClusterInfo createClusterInfo(Map diskUsage, Map shardSizes) { - return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), Map.of()); + return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), Map.of(), Map.of()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java index 7e05ae7c57f79..8ab031aa53fe1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java @@ -608,6 +608,7 @@ public void testShardSizeDiscrepancyWithinIndex() { ), Map.of(), Map.of(), + Map.of(), Map.of() ) ); @@ -704,7 +705,7 @@ private RoutingAllocation createRoutingAllocation(ClusterState clusterState) { } private static ClusterInfo createClusterInfo(Map indexSizes) { - return new ClusterInfo(Map.of(), Map.of(), indexSizes, Map.of(), Map.of(), Map.of()); + return new ClusterInfo(Map.of(), Map.of(), indexSizes, Map.of(), Map.of(), Map.of(), Map.of()); } private static IndexMetadata.Builder anIndex(String name) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java index c75912fda27e3..277521c5832a1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java @@ -561,7 +561,7 @@ public ClusterInfo getClusterInfo() { dataPath.put(new ClusterInfo.NodeAndShard(shardRouting.currentNodeId(), shardRouting.shardId()), "/data"); } - return new ClusterInfo(diskSpaceUsage, diskSpaceUsage, shardSizes, Map.of(), dataPath, Map.of()); + return new ClusterInfo(diskSpaceUsage, diskSpaceUsage, shardSizes, Map.of(), dataPath, Map.of(), Map.of()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java index 6adb9f1e8ad26..80fe603488fd3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java @@ -345,6 +345,7 @@ private ClusterInfo createClusterInfo(List> shardSizes) { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), Map.of(), Map.of(), + Map.of(), Map.of() ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterInfoSimulatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterInfoSimulatorTests.java index cdb8c5f602032..b67e248999ced 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterInfoSimulatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterInfoSimulatorTests.java @@ -690,7 +690,15 @@ public ClusterInfoTestBuilder withReservedSpace(String nodeId, String path, long } public ClusterInfo build() { - return new ClusterInfo(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of(), reservedSpace); + return new ClusterInfo( + leastAvailableSpaceUsage, + mostAvailableSpaceUsage, + shardSizes, + Map.of(), + Map.of(), + reservedSpace, + Map.of() + ); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java index 6e496b85ede97..a0d28ce124584 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java @@ -690,7 +690,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() { .stream() .collect(toMap(Map.Entry::getKey, it -> new DiskUsage(it.getKey(), it.getKey(), "/data", diskSize, diskSize - it.getValue()))); - var clusterInfo = new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), dataPath, Map.of()); + var clusterInfo = new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), dataPath, Map.of(), Map.of()); var settings = Settings.EMPTY; @@ -1196,7 +1196,7 @@ public ClusterInfoTestBuilder withReservedSpace(String nodeId, long size, ShardI } public ClusterInfo build() { - return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), reservedSpace); + return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), reservedSpace, Map.of()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java index 4eed552d5f1af..844912cba4c17 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java @@ -619,6 +619,7 @@ public void testUnassignedAllocationPredictsDiskUsage() { shardSizesBuilder.build(), ImmutableOpenMap.of(), ImmutableOpenMap.of(), + ImmutableOpenMap.of(), ImmutableOpenMap.of() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 9522629de8f0d..5467d313834b8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -1406,7 +1406,7 @@ static class DevNullClusterInfo extends ClusterInfo { Map shardSizes, Map reservedSpace ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of(), reservedSpace); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of(), reservedSpace, Map.of()); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index d8cb13d7a7ba2..7da75f61da801 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -108,6 +108,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { Map.of("[test][0][p]", 10L), // 10 bytes, Map.of(), Map.of(), + Map.of(), Map.of() ); RoutingAllocation allocation = new RoutingAllocation( @@ -179,6 +180,7 @@ private void doTestCannotAllocateDueToLackOfDiskResources(boolean testMaxHeadroo Map.of("[test][0][p]", shardSize), Map.of(), Map.of(), + Map.of(), Map.of() ); RoutingAllocation allocation = new RoutingAllocation( @@ -324,6 +326,7 @@ private void doTestCanRemainUsesLeastAvailableSpace(boolean testMaxHeadroom) { shardSizes, Map.of(), shardRoutingMap, + Map.of(), Map.of() ); RoutingAllocation allocation = new RoutingAllocation( @@ -843,6 +846,7 @@ public void testDecidesYesIfWatermarksIgnored() { Map.of("[test][0][p]", 10L), Map.of(), Map.of(), + Map.of(), Map.of() ); RoutingAllocation allocation = new RoutingAllocation( @@ -904,7 +908,15 @@ public void testCannotForceAllocateOver100PercentUsage() { // bigger than available space final long shardSize = randomIntBetween(1, 10); shardSizes.put("[test][0][p]", shardSize); - ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of(), Map.of()); + ClusterInfo clusterInfo = new ClusterInfo( + leastAvailableUsages, + mostAvailableUsage, + shardSizes, + Map.of(), + Map.of(), + Map.of(), + Map.of() + ); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState, diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 77ba019835ec9..e451b1d45817d 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -959,6 +959,7 @@ private ExtendedClusterInfo(Map extraShardSizes, ClusterInfo info) extraShardSizes, Map.of(), Map.of(), + Map.of(), Map.of() ); this.delegate = info; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 4061d37832184..12f7dde103c9c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -262,7 +262,7 @@ public void testContext() { } } state = ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build(); - info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, @@ -311,7 +311,7 @@ public void testContext() { ) ); - info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java index ab09f20e34397..37295ebf44208 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java @@ -109,7 +109,7 @@ public Tuple sizeAndClusterInfo(IndexMetadata indexMetadata) // add irrelevant shards noise for completeness (should not happen IRL). sizes.put(new ShardId(index, i), randomLongBetween(0, Integer.MAX_VALUE)); } - ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), Map.of(), sizes, Map.of(), Map.of()); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), Map.of(), sizes, Map.of(), Map.of(), Map.of()); return Tuple.tuple(totalSize, info); } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java index 1960b7f2028e7..b252fdf5564db 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java @@ -397,7 +397,7 @@ private ClusterInfo randomClusterInfo(ClusterState state) { for (var id : state.nodes().getDataNodes().keySet()) { diskUsage.put(id, new DiskUsage(id, id, "/test", Long.MAX_VALUE, Long.MAX_VALUE)); } - return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), Map.of()); + return new ClusterInfo(diskUsage, diskUsage, shardSizes, Map.of(), Map.of(), Map.of(), Map.of()); } private ClusterState.Builder applyCreatedDates( diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index d159e67482695..2ee94340f6d2c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -379,7 +379,7 @@ public void validateSizeOf(ClusterState clusterState, ShardRouting subjectShard, } private ReactiveStorageDeciderService.AllocationState createAllocationState(Map shardSize, ClusterState clusterState) { - ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), shardSize, Map.of(), Map.of(), Map.of()); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), shardSize, Map.of(), Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, null, @@ -544,7 +544,7 @@ public void testUnmovableSize() { } var diskUsages = Map.of(nodeId, new DiskUsage(nodeId, null, null, ByteSizeUnit.KB.toBytes(100), ByteSizeUnit.KB.toBytes(5))); - ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of()); + ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index a0a37f2bb52d1..40a564088aee6 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -173,6 +173,7 @@ public void testCheckDiskLowWatermark() { Map.of(), Map.of(), Map.of(), + Map.of(), Map.of() ); DeprecationIssue issue = TransportNodeDeprecationCheckAction.checkDiskLowWatermark( From 542c256aba5c1d295f46c14e13241161ab691919 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Mon, 2 Jun 2025 15:04:50 +1000 Subject: [PATCH 02/30] Populate nodesHeapUsage, make HeapUsageSupplier pluggable --- .../elasticsearch/cluster/ClusterInfo.java | 26 +++++++++----- .../org/elasticsearch/cluster/HeapUsage.java | 11 ++++-- .../cluster/HeapUsageSupplier.java | 29 ++++++++++++++++ .../cluster/InternalClusterInfoService.java | 34 ++++++++++++++++--- .../node/NodeServiceProvider.java | 24 ++++++++++++- .../elasticsearch/plugins/ClusterPlugin.java | 11 ++++++ ...rnalClusterInfoServiceSchedulingTests.java | 8 ++++- .../MockInternalClusterInfoService.java | 2 +- 8 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index be1d8ff51694a..86400859baef8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -36,7 +36,7 @@ import static org.elasticsearch.cluster.routing.ShardRouting.newUnassigned; import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; /** @@ -57,7 +57,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { final Map shardDataSetSizes; final Map dataPath; final Map reservedSpace; - final Map nodeHeapUsage; + final Map nodesHeapUsage; protected ClusterInfo() { this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); @@ -81,7 +81,7 @@ public ClusterInfo( Map shardDataSetSizes, Map dataPath, Map reservedSpace, - Map nodeHeapUsage + Map nodesHeapUsage ) { this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage); this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage); @@ -89,7 +89,7 @@ public ClusterInfo( this.shardDataSetSizes = Map.copyOf(shardDataSetSizes); this.dataPath = Map.copyOf(dataPath); this.reservedSpace = Map.copyOf(reservedSpace); - this.nodeHeapUsage = Map.copyOf(nodeHeapUsage); + this.nodesHeapUsage = Map.copyOf(nodesHeapUsage); } public ClusterInfo(StreamInput in) throws IOException { @@ -102,9 +102,9 @@ public ClusterInfo(StreamInput in) throws IOException { : in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString); this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new); if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { - this.nodeHeapUsage = in.readImmutableMap(HeapUsage::new); + this.nodesHeapUsage = in.readImmutableMap(HeapUsage::new); } else { - this.nodeHeapUsage = Map.of(); + this.nodesHeapUsage = Map.of(); } } @@ -121,7 +121,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeMap(this.reservedSpace); if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { - out.writeMap(this.nodeHeapUsage, StreamOutput::writeWriteable); + out.writeMap(this.nodesHeapUsage, StreamOutput::writeWriteable); } } @@ -202,7 +202,17 @@ public Iterator toXContentChunked(ToXContent.Params params } return builder.endObject(); // NodeAndPath }), - endArray() // end "reserved_sizes" + chunk( + (builder, p) -> builder.endArray() // end "reserved_sizes" + .startObject("heap_usage") + ), + Iterators.map(nodesHeapUsage.entrySet().iterator(), c -> (builder, p) -> { + builder.startObject(c.getKey()); + c.getValue().toShortXContent(builder); + builder.endObject(); + return builder; + }), + endObject() // end "heap_usage" ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java index d28c80cd3bf54..7cdbc6c202fea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java @@ -35,9 +35,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(this.freeBytes); } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("node_id", this.nodeId); + public XContentBuilder toShortXContent(XContentBuilder builder) throws IOException { builder.field("node_name", this.nodeName); builder.humanReadableField("total_heap_bytes", "total", ByteSizeValue.ofBytes(this.totalBytes)); builder.humanReadableField("used_heap_bytes", "used", ByteSizeValue.ofBytes(this.usedBytes())); @@ -47,6 +45,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("node_id", this.nodeId); + toShortXContent(builder); + return builder; + } + public double freeHeapAsPercentage() { return 100.0 * freeBytes / (double) totalBytes; } diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java b/server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java new file mode 100644 index 0000000000000..54bca56d376b2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.action.ActionListener; + +import java.util.Map; + +public interface HeapUsageSupplier { + + /** + * This will be used when there are no heap usage suppliers available + */ + HeapUsageSupplier EMPTY = listener -> listener.onResponse(Map.of()); + + /** + * Get the heap usage for every node in the cluster + * + * @param listener The listener which will receive the results + */ + void getClusterHeapUsage(ActionListener> listener); +} diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 29d699ec26d48..4e0a7562f369a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -89,11 +89,12 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; private volatile IndicesStatsSummary indicesStatsSummary; - private volatile Map nodeHeapUsages; + private volatile Map nodesHeapUsage; private final ThreadPool threadPool; private final Client client; private final List> listeners = new CopyOnWriteArrayList<>(); + private final HeapUsageSupplier heapUsageSupplier; private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); @@ -102,13 +103,20 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private RefreshScheduler refreshScheduler; @SuppressWarnings("this-escape") - public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { + public InternalClusterInfoService( + Settings settings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + HeapUsageSupplier heapUsageSupplier + ) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); - this.nodeHeapUsages = Map.of(); + this.nodesHeapUsage = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; + this.heapUsageSupplier = heapUsageSupplier; this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings); this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings); this.enabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); @@ -175,6 +183,7 @@ void execute() { logger.trace("skipping collecting info from cluster, notifying listeners with empty cluster info"); leastAvailableSpaceUsages = Map.of(); mostAvailableSpaceUsages = Map.of(); + nodesHeapUsage = Map.of(); indicesStatsSummary = IndicesStatsSummary.EMPTY; callListeners(); return; @@ -189,9 +198,26 @@ void execute() { try (var ignored = threadPool.getThreadContext().clearTraceContext()) { fetchIndicesStats(); } + try (var ignored = threadPool.getThreadContext().clearTraceContext()) { + fetchNodesHeapUsage(); + } } } + private void fetchNodesHeapUsage() { + heapUsageSupplier.getClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { + @Override + public void onResponse(Map stringHeapUsageMap) { + nodesHeapUsage = stringHeapUsageMap; + } + + @Override + public void onFailure(Exception e) { + logger.warn("failed to fetch heap usage for nodes", e); + } + }, fetchRefs.acquire())); + } + private void fetchIndicesStats() { final IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); indicesStatsRequest.clear(); @@ -416,7 +442,7 @@ public ClusterInfo getClusterInfo() { indicesStatsSummary.shardDataSetSizes, indicesStatsSummary.dataPath, indicesStatsSummary.reservedSpace, - nodeHeapUsages + nodesHeapUsage ); } diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 1c3f19b20b4dc..26d9fe622c47d 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.OnlinePrewarmingService; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterInfoService; +import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -28,6 +29,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.PluginsLoader; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.ReadinessService; @@ -44,6 +46,7 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; +import java.util.Objects; import java.util.function.Function; import java.util.function.LongSupplier; @@ -74,7 +77,14 @@ ClusterInfoService newClusterInfoService( ThreadPool threadPool, NodeClient client ) { - final InternalClusterInfoService service = new InternalClusterInfoService(settings, clusterService, threadPool, client); + final HeapUsageSupplier heapUsageSupplier = getHeapUsageSupplier(pluginsService); + final InternalClusterInfoService service = new InternalClusterInfoService( + settings, + clusterService, + threadPool, + client, + heapUsageSupplier + ); if (DiscoveryNode.isMasterNode(settings)) { // listen for state changes (this node starts/stops being the elected master, or new nodes are added) clusterService.addListener(service); @@ -146,4 +156,16 @@ void processRecoverySettings(PluginsService pluginsService, ClusterSettings clus ReadinessService newReadinessService(PluginsService pluginsService, ClusterService clusterService, Environment environment) { return new ReadinessService(clusterService, environment); } + + private static HeapUsageSupplier getHeapUsageSupplier(PluginsService pluginsService) { + final var heapUsageSuppliers = pluginsService.filterPlugins(ClusterPlugin.class) + .map(ClusterPlugin::getHeapUsageSupplier) + .filter(Objects::nonNull) + .toList(); + return switch (heapUsageSuppliers.size()) { + case 0 -> HeapUsageSupplier.EMPTY; + case 1 -> heapUsageSuppliers.getFirst(); + default -> throw new IllegalArgumentException("multiple plugins define heap usage suppliers, which is not permitted"); + }; + } } diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 3f71b901ee32f..7a6d5fdbf868f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -9,6 +9,7 @@ package org.elasticsearch.plugins; +import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; @@ -41,6 +42,16 @@ default Collection createAllocationDeciders(Settings settings return Collections.emptyList(); } + /** + * Create a {@link HeapUsageSupplier} that will be used to determine the approximate heap usage for all + * cluster nodes + *

+ * Note: Only a single {@link ClusterPlugin} can define a heap usage supplier. + */ + default HeapUsageSupplier getHeapUsageSupplier() { + return null; + } + /** * Return {@link ShardsAllocator} implementations added by this plugin. * diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index bcf5be82f6da7..462907b32278a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -71,7 +71,13 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final ClusterService clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService); final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); - final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService(settings, clusterService, threadPool, client); + final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService( + settings, + clusterService, + threadPool, + client, + HeapUsageSupplier.EMPTY + ); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 49a36f64e2811..4df87ed6ea6bf 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -43,7 +43,7 @@ public static class TestPlugin extends Plugin {} private volatile BiFunction diskUsageFunction; public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) { - super(settings, clusterService, threadPool, client); + super(settings, clusterService, threadPool, client, HeapUsageSupplier.EMPTY); } public void setDiskUsageFunctionAndRefresh(BiFunction diskUsageFn) { From 356beb5b6edd0c70a7226ab47aa008e1cf0c0f82 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Mon, 2 Jun 2025 15:35:32 +1000 Subject: [PATCH 03/30] Fix tests --- .../src/main/java/org/elasticsearch/cluster/ClusterInfo.java | 3 ++- .../admin/cluster/allocation/DesiredBalanceResponseTests.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index 86400859baef8..7a07711780f3f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -322,7 +322,8 @@ public String toString() { // exposed for tests, computed here rather than exposing all the collections separately int getChunkCount() { - return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 6; + return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + + nodesHeapUsage.size() + 7; } public record NodeAndShard(String nodeId, ShardId shardId) implements Writeable { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java index 059605457136b..0961d7bff536d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java @@ -337,7 +337,7 @@ public void testToXContent() throws IOException { Map clusterInfo = (Map) json.get("cluster_info"); assertThat( clusterInfo.keySet(), - containsInAnyOrder("nodes", "shard_paths", "shard_sizes", "shard_data_set_sizes", "reserved_sizes") + containsInAnyOrder("nodes", "shard_paths", "shard_sizes", "shard_data_set_sizes", "reserved_sizes", "heap_usage") ); } From 81fd063827ce9559371bf79f46978fce2cc54937 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 14:05:59 +1000 Subject: [PATCH 04/30] Allow deferred creation of HeapUsageSupplier --- .../cluster/InternalClusterInfoService.java | 21 +++++++++++-------- .../elasticsearch/node/NodeConstruction.java | 13 ++++++------ .../node/NodeServiceProvider.java | 11 ++-------- ...rnalClusterInfoServiceSchedulingTests.java | 8 +------ .../MockInternalClusterInfoService.java | 2 +- 5 files changed, 22 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 4e0a7562f369a..2d9c545be1256 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -94,29 +94,22 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private final ThreadPool threadPool; private final Client client; private final List> listeners = new CopyOnWriteArrayList<>(); - private final HeapUsageSupplier heapUsageSupplier; private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); + private HeapUsageSupplier heapUsageSupplier; private AsyncRefresh currentRefresh; private RefreshScheduler refreshScheduler; @SuppressWarnings("this-escape") - public InternalClusterInfoService( - Settings settings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - HeapUsageSupplier heapUsageSupplier - ) { + public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); this.nodesHeapUsage = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; - this.heapUsageSupplier = heapUsageSupplier; this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings); this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings); this.enabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); @@ -141,6 +134,16 @@ void setUpdateFrequency(TimeValue updateFrequency) { this.updateFrequency = updateFrequency; } + /** + * This can be provided by plugins, which are initialised long after the ClusterInfoService is created + * + * @param heapUsageSupplier The HeapUsageSupplier to use + */ + public void setHeapUsageSupplier(HeapUsageSupplier heapUsageSupplier) { + assert this.heapUsageSupplier == null; + this.heapUsageSupplier = heapUsageSupplier; + } + @Override public void clusterChanged(ClusterChangedEvent event) { final Runnable newRefresh; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index eb4f2c2543475..373a0fd38280b 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService; import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.MasterHistoryService; @@ -340,6 +341,7 @@ static NodeConstruction prepareConstruction( private TerminationHandler terminationHandler; private NamedWriteableRegistry namedWriteableRegistry; private NamedXContentRegistry xContentRegistry; + private ClusterInfoService clusterInfoService; private NodeConstruction(List resourcesToClose) { this.resourcesToClose = resourcesToClose; @@ -733,13 +735,7 @@ private void construct( ); RepositoriesService repositoriesService = repositoriesModule.getRepositoryService(); final SetOnce rerouteServiceReference = new SetOnce<>(); - final ClusterInfoService clusterInfoService = serviceProvider.newClusterInfoService( - pluginsService, - settings, - clusterService, - threadPool, - client - ); + this.clusterInfoService = serviceProvider.newClusterInfoService(pluginsService, settings, clusterService, threadPool, client); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, clusterService, @@ -990,6 +986,9 @@ public Map queryFields() { .map(TerminationHandlerProvider::handler); terminationHandler = getSinglePlugin(terminationHandlers, TerminationHandler.class).orElse(null); + if (clusterInfoService instanceof InternalClusterInfoService icis) { + icis.setHeapUsageSupplier(serviceProvider.newHeapUsageSupplier(pluginsService)); + } final IncrementalBulkService incrementalBulkService = new IncrementalBulkService(client, indexingLimits); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 26d9fe622c47d..35ef3731674d0 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -77,14 +77,7 @@ ClusterInfoService newClusterInfoService( ThreadPool threadPool, NodeClient client ) { - final HeapUsageSupplier heapUsageSupplier = getHeapUsageSupplier(pluginsService); - final InternalClusterInfoService service = new InternalClusterInfoService( - settings, - clusterService, - threadPool, - client, - heapUsageSupplier - ); + final InternalClusterInfoService service = new InternalClusterInfoService(settings, clusterService, threadPool, client); if (DiscoveryNode.isMasterNode(settings)) { // listen for state changes (this node starts/stops being the elected master, or new nodes are added) clusterService.addListener(service); @@ -157,7 +150,7 @@ ReadinessService newReadinessService(PluginsService pluginsService, ClusterServi return new ReadinessService(clusterService, environment); } - private static HeapUsageSupplier getHeapUsageSupplier(PluginsService pluginsService) { + HeapUsageSupplier newHeapUsageSupplier(PluginsService pluginsService) { final var heapUsageSuppliers = pluginsService.filterPlugins(ClusterPlugin.class) .map(ClusterPlugin::getHeapUsageSupplier) .filter(Objects::nonNull) diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index 462907b32278a..bcf5be82f6da7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -71,13 +71,7 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final ClusterService clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService); final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); - final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService( - settings, - clusterService, - threadPool, - client, - HeapUsageSupplier.EMPTY - ); + final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService(settings, clusterService, threadPool, client); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 4df87ed6ea6bf..49a36f64e2811 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -43,7 +43,7 @@ public static class TestPlugin extends Plugin {} private volatile BiFunction diskUsageFunction; public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) { - super(settings, clusterService, threadPool, client, HeapUsageSupplier.EMPTY); + super(settings, clusterService, threadPool, client); } public void setDiskUsageFunctionAndRefresh(BiFunction diskUsageFn) { From bc0682cd2646ff526916b9f8415bdacd9efd850a Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 14:37:56 +1000 Subject: [PATCH 05/30] Default HeapUsageSupplier --- .../org/elasticsearch/cluster/InternalClusterInfoService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 2d9c545be1256..551a4ce36c0ca 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -98,7 +98,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); - private HeapUsageSupplier heapUsageSupplier; + private HeapUsageSupplier heapUsageSupplier = HeapUsageSupplier.EMPTY; private AsyncRefresh currentRefresh; private RefreshScheduler refreshScheduler; @@ -140,7 +140,7 @@ void setUpdateFrequency(TimeValue updateFrequency) { * @param heapUsageSupplier The HeapUsageSupplier to use */ public void setHeapUsageSupplier(HeapUsageSupplier heapUsageSupplier) { - assert this.heapUsageSupplier == null; + assert this.heapUsageSupplier == HeapUsageSupplier.EMPTY; this.heapUsageSupplier = heapUsageSupplier; } From 13d1de804a45dcd4e942c4b25e3ca07486a46fa1 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 15:23:28 +1000 Subject: [PATCH 06/30] Clarify that heap usage is a minimum --- .../java/org/elasticsearch/cluster/ClusterInfo.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index 7a07711780f3f..7ce9d2f477ca7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -72,6 +72,7 @@ protected ClusterInfo() { * @param shardDataSetSizes a shard id to data set size in bytes mapping per shard * @param dataPath the shard routing to datapath mapping * @param reservedSpace reserved space per shard broken down by node and data path + * @param nodesHeapUsage heap usage broken down by node * @see #shardIdentifierFromRouting */ public ClusterInfo( @@ -216,6 +217,15 @@ public Iterator toXContentChunked(ToXContent.Params params ); } + /** + * Returns a node id to estimated heap usage mapping for all nodes that we have such data for. + * Note that these estimates should be considered minimums. They may be used to determine whether + * there IS NOT capacity to do something, but not to determine that there IS capacity to do something. + */ + public Map getNodesHeapUsage() { + return nodesHeapUsage; + } + /** * Returns a node id to disk usage mapping for the path that has the least available space on the node. * Note that this does not take account of reserved space: there may be another path with less available _and unreserved_ space. From f4d9db51bdb3a6784002e89a2348f9b8a7770b25 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 16:18:08 +1000 Subject: [PATCH 07/30] Test that InternalClusterInfoService polls for heap usage --- ...ternalClusterInfoServiceSchedulingTests.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index bcf5be82f6da7..24593e3a7c602 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -34,12 +34,17 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.ThreadPool; +import org.mockito.Mockito; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; public class InternalClusterInfoServiceSchedulingTests extends ESTestCase { @@ -72,6 +77,8 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService(settings, clusterService, threadPool, client); + final HeapUsageSupplier mockHeapUsageSupplier = spy(new StubHeapUsageSupplier()); + clusterInfoService.setHeapUsageSupplier(mockHeapUsageSupplier); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); @@ -107,11 +114,13 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { deterministicTaskQueue.runAllRunnableTasks(); for (int i = 0; i < 3; i++) { + Mockito.clearInvocations(mockHeapUsageSupplier); final int initialRequestCount = client.requestCount; final long duration = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings).millis(); runFor(deterministicTaskQueue, duration); deterministicTaskQueue.runAllRunnableTasks(); assertThat(client.requestCount, equalTo(initialRequestCount + 2)); // should have run two client requests per interval + verify(mockHeapUsageSupplier).getClusterHeapUsage(any()); // Should poll for heap usage once per interval } final AtomicBoolean failMaster2 = new AtomicBoolean(); @@ -128,6 +137,14 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { assertFalse(deterministicTaskQueue.hasDeferredTasks()); } + private static class StubHeapUsageSupplier implements HeapUsageSupplier { + + @Override + public void getClusterHeapUsage(ActionListener> listener) { + listener.onResponse(Map.of()); + } + } + private static void runFor(DeterministicTaskQueue deterministicTaskQueue, long duration) { final long endTime = deterministicTaskQueue.getCurrentTimeMillis() + duration; while (deterministicTaskQueue.getCurrentTimeMillis() < endTime From bf51e859bcb11204315ca63f4c1702b9539942fd Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 16:55:15 +1000 Subject: [PATCH 08/30] Test that getNodesHeapUsage returns heap usage --- .../index/shard/IndexShardIT.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 507724bbaeb3c..1d716445fc362 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.HeapUsage; +import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -61,6 +63,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.DummyShardLock; @@ -81,6 +84,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; @@ -89,6 +93,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLength; @@ -110,12 +115,13 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class IndexShardIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class); + return pluginList(InternalSettingsPlugin.class, BogusHeapUsagePlugin.class); } public void testLockTryingToDelete() throws Exception { @@ -253,6 +259,20 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException { assertThat(dataSetSize.get(), greaterThan(0L)); } + public void testHeapUsageEstimateIsPresent() { + InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); + ClusterInfoServiceUtils.refresh(clusterInfoService); + ClusterState state = getInstanceFromNode(ClusterService.class).state(); + Map heapUsages = clusterInfoService.getClusterInfo().getNodesHeapUsage(); + assertNotNull(heapUsages); + assertEquals(state.nodes().size(), heapUsages.size()); + for (DiscoveryNode node : state.nodes()) { + assertTrue(heapUsages.containsKey(node.getId())); + HeapUsage heapUsage = heapUsages.get(node.getId()); + assertThat(heapUsage.freeBytes(), lessThanOrEqualTo(heapUsage.totalBytes())); + } + } + public void testIndexCanChangeCustomDataPath() throws Exception { final String index = "test-custom-data-path"; final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataDir().resolve(randomAsciiLettersOfLength(10)); @@ -795,4 +815,43 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable assertFalse(indicesService.hasUncompletedPendingDeletes()), 1, TimeUnit.MINUTES); } } + + private static class BogusHeapUsageSupplier implements HeapUsageSupplier { + + private final ClusterService clusterService; + + private BogusHeapUsageSupplier(ClusterService clusterService) { + this.clusterService = clusterService; + } + + @Override + public void getClusterHeapUsage(ActionListener> listener) { + ActionListener.completeWith( + listener, + () -> clusterService.state().nodes().stream().collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { + final long maxHeap = randomNonNegativeLong(); + final long freeHeap = (long) (randomFloat() * maxHeap); + return new HeapUsage(node.getId(), node.getName(), maxHeap, freeHeap); + })) + ); + } + } + + public static class BogusHeapUsagePlugin extends Plugin implements ClusterPlugin { + + private BogusHeapUsageSupplier bogusHeapUsageSupplier; + + public BogusHeapUsagePlugin() {} + + @Override + public Collection createComponents(PluginServices services) { + bogusHeapUsageSupplier = new BogusHeapUsageSupplier(services.clusterService()); + return Collections.singletonList(bogusHeapUsageSupplier); + } + + @Override + public HeapUsageSupplier getHeapUsageSupplier() { + return bogusHeapUsageSupplier; + } + } } From c47c0caa16e3bfa35835bde973a0083a5273b136 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 3 Jun 2025 17:07:28 +1000 Subject: [PATCH 09/30] More caveats for #getNodesHeapUsage() --- server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index 7ce9d2f477ca7..c97a47083a9a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -221,6 +221,8 @@ public Iterator toXContentChunked(ToXContent.Params params * Returns a node id to estimated heap usage mapping for all nodes that we have such data for. * Note that these estimates should be considered minimums. They may be used to determine whether * there IS NOT capacity to do something, but not to determine that there IS capacity to do something. + * Also note that the map may not be complete, it may contain none, or a subset of the nodes in + * the cluster at any time. It may also contain entries for nodes that have since left the cluster. */ public Map getNodesHeapUsage() { return nodesHeapUsage; From 23eb8e61036f7ef0b9e6246c9b975d3dded01ed6 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 4 Jun 2025 10:48:37 +1000 Subject: [PATCH 10/30] Remove HeapUsageSupplier from ClusterPlugin interface --- .../elasticsearch/index/shard/IndexShardIT.java | 12 +++--------- .../routing/allocation/AllocationService.java | 8 ++++++++ .../org/elasticsearch/node/NodeConstruction.java | 13 +++++++------ .../elasticsearch/node/NodeServiceProvider.java | 15 --------------- .../org/elasticsearch/plugins/ClusterPlugin.java | 11 ----------- 5 files changed, 18 insertions(+), 41 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 1d716445fc362..83f5bab416734 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -839,19 +839,13 @@ public void getClusterHeapUsage(ActionListener> listener) public static class BogusHeapUsagePlugin extends Plugin implements ClusterPlugin { - private BogusHeapUsageSupplier bogusHeapUsageSupplier; - public BogusHeapUsagePlugin() {} @Override public Collection createComponents(PluginServices services) { - bogusHeapUsageSupplier = new BogusHeapUsageSupplier(services.clusterService()); - return Collections.singletonList(bogusHeapUsageSupplier); - } - - @Override - public HeapUsageSupplier getHeapUsageSupplier() { - return bogusHeapUsageSupplier; + BogusHeapUsageSupplier bogusHeapUsageSupplier = new BogusHeapUsageSupplier(services.clusterService()); + services.allocationService().setHeapUsageSupplier(bogusHeapUsageSupplier); + return List.of(); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 263c2362c9ed6..b685896933d0f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -15,6 +15,8 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.HeapUsageSupplier; +import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -637,6 +639,12 @@ public void addAllocFailuresResetListenerTo(ClusterService clusterService) { }); } + public void setHeapUsageSupplier(HeapUsageSupplier heapUsageSupplier) { + if (clusterInfoService instanceof InternalClusterInfoService icis) { + icis.setHeapUsageSupplier(heapUsageSupplier); + } + } + /** * We should reset allocation/relocation failure count to allow further retries when: * diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 373a0fd38280b..eb4f2c2543475 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService; import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.MasterHistoryService; @@ -341,7 +340,6 @@ static NodeConstruction prepareConstruction( private TerminationHandler terminationHandler; private NamedWriteableRegistry namedWriteableRegistry; private NamedXContentRegistry xContentRegistry; - private ClusterInfoService clusterInfoService; private NodeConstruction(List resourcesToClose) { this.resourcesToClose = resourcesToClose; @@ -735,7 +733,13 @@ private void construct( ); RepositoriesService repositoriesService = repositoriesModule.getRepositoryService(); final SetOnce rerouteServiceReference = new SetOnce<>(); - this.clusterInfoService = serviceProvider.newClusterInfoService(pluginsService, settings, clusterService, threadPool, client); + final ClusterInfoService clusterInfoService = serviceProvider.newClusterInfoService( + pluginsService, + settings, + clusterService, + threadPool, + client + ); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, clusterService, @@ -986,9 +990,6 @@ public Map queryFields() { .map(TerminationHandlerProvider::handler); terminationHandler = getSinglePlugin(terminationHandlers, TerminationHandler.class).orElse(null); - if (clusterInfoService instanceof InternalClusterInfoService icis) { - icis.setHeapUsageSupplier(serviceProvider.newHeapUsageSupplier(pluginsService)); - } final IncrementalBulkService incrementalBulkService = new IncrementalBulkService(client, indexingLimits); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 35ef3731674d0..1c3f19b20b4dc 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.search.OnlinePrewarmingService; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterInfoService; -import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -29,7 +28,6 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.PluginsLoader; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.ReadinessService; @@ -46,7 +44,6 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; -import java.util.Objects; import java.util.function.Function; import java.util.function.LongSupplier; @@ -149,16 +146,4 @@ void processRecoverySettings(PluginsService pluginsService, ClusterSettings clus ReadinessService newReadinessService(PluginsService pluginsService, ClusterService clusterService, Environment environment) { return new ReadinessService(clusterService, environment); } - - HeapUsageSupplier newHeapUsageSupplier(PluginsService pluginsService) { - final var heapUsageSuppliers = pluginsService.filterPlugins(ClusterPlugin.class) - .map(ClusterPlugin::getHeapUsageSupplier) - .filter(Objects::nonNull) - .toList(); - return switch (heapUsageSuppliers.size()) { - case 0 -> HeapUsageSupplier.EMPTY; - case 1 -> heapUsageSuppliers.getFirst(); - default -> throw new IllegalArgumentException("multiple plugins define heap usage suppliers, which is not permitted"); - }; - } } diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java index 7a6d5fdbf868f..3f71b901ee32f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java @@ -9,7 +9,6 @@ package org.elasticsearch.plugins; -import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; @@ -42,16 +41,6 @@ default Collection createAllocationDeciders(Settings settings return Collections.emptyList(); } - /** - * Create a {@link HeapUsageSupplier} that will be used to determine the approximate heap usage for all - * cluster nodes - *

- * Note: Only a single {@link ClusterPlugin} can define a heap usage supplier. - */ - default HeapUsageSupplier getHeapUsageSupplier() { - return null; - } - /** * Return {@link ShardsAllocator} implementations added by this plugin. * From 887bcafa0c2daf2f5be59d564755250ee4e99b70 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 4 Jun 2025 11:01:59 +1000 Subject: [PATCH 11/30] Swap free for used in HeapUsage --- .../index/shard/IndexShardIT.java | 4 ++-- .../org/elasticsearch/cluster/HeapUsage.java | 20 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 83f5bab416734..b101ab288d50a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -830,8 +830,8 @@ public void getClusterHeapUsage(ActionListener> listener) listener, () -> clusterService.state().nodes().stream().collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { final long maxHeap = randomNonNegativeLong(); - final long freeHeap = (long) (randomFloat() * maxHeap); - return new HeapUsage(node.getId(), node.getName(), maxHeap, freeHeap); + final long usedHeap = (long) (randomFloat() * maxHeap); + return new HeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); })) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java index 7cdbc6c202fea..f4bdc77268cfe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java @@ -21,7 +21,11 @@ /** * Record representing the heap usage for a single cluster node */ -public record HeapUsage(String nodeId, String nodeName, long totalBytes, long freeBytes) implements ToXContentFragment, Writeable { +public record HeapUsage(String nodeId, String nodeName, long totalBytes, long usedBytes) implements ToXContentFragment, Writeable { + + public HeapUsage { + assert usedBytes <= totalBytes; + } public HeapUsage(StreamInput in) throws IOException { this(in.readString(), in.readString(), in.readVLong(), in.readVLong()); @@ -32,14 +36,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(this.nodeId); out.writeString(this.nodeName); out.writeVLong(this.totalBytes); - out.writeVLong(this.freeBytes); + out.writeVLong(this.usedBytes); } public XContentBuilder toShortXContent(XContentBuilder builder) throws IOException { builder.field("node_name", this.nodeName); builder.humanReadableField("total_heap_bytes", "total", ByteSizeValue.ofBytes(this.totalBytes)); - builder.humanReadableField("used_heap_bytes", "used", ByteSizeValue.ofBytes(this.usedBytes())); - builder.humanReadableField("free_heap_bytes", "free", ByteSizeValue.ofBytes(this.freeBytes)); + builder.humanReadableField("used_heap_bytes", "used", ByteSizeValue.ofBytes(this.usedBytes)); + builder.humanReadableField("free_heap_bytes", "free", ByteSizeValue.ofBytes(this.freeBytes())); builder.field("free_heap_percent", truncatePercent(this.freeHeapAsPercentage())); builder.field("used_heap_percent", truncatePercent(this.usedHeapAsPercentage())); return builder; @@ -53,15 +57,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public double freeHeapAsPercentage() { - return 100.0 * freeBytes / (double) totalBytes; + return 100.0 - usedHeapAsPercentage(); } public double usedHeapAsPercentage() { - return 100.0 - freeHeapAsPercentage(); + return 100.0 * usedBytes / (double) totalBytes; } - public long usedBytes() { - return totalBytes - freeBytes; + public long freeBytes() { + return totalBytes - usedBytes; } private static double truncatePercent(double pct) { From 85fd019c84e47f2ca09c5c071eeeb5d297493c4f Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 4 Jun 2025 11:25:07 +1000 Subject: [PATCH 12/30] Don't report heap usage in ClusterInfo serialization --- .../org/elasticsearch/cluster/ClusterInfo.java | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index c97a47083a9a9..fc981d97e7efc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -36,7 +36,7 @@ import static org.elasticsearch.cluster.routing.ShardRouting.newUnassigned; import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; /** @@ -203,17 +203,7 @@ public Iterator toXContentChunked(ToXContent.Params params } return builder.endObject(); // NodeAndPath }), - chunk( - (builder, p) -> builder.endArray() // end "reserved_sizes" - .startObject("heap_usage") - ), - Iterators.map(nodesHeapUsage.entrySet().iterator(), c -> (builder, p) -> { - builder.startObject(c.getKey()); - c.getValue().toShortXContent(builder); - builder.endObject(); - return builder; - }), - endObject() // end "heap_usage" + endArray() // end "reserved_sizes" ); } @@ -334,8 +324,7 @@ public String toString() { // exposed for tests, computed here rather than exposing all the collections separately int getChunkCount() { - return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() - + nodesHeapUsage.size() + 7; + return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 6; } public record NodeAndShard(String nodeId, ShardId shardId) implements Writeable { From f112a3b49cfe87b7cedbd80201da0d3d6d1454e5 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 4 Jun 2025 13:01:09 +1000 Subject: [PATCH 13/30] Fix tests --- .../admin/cluster/allocation/DesiredBalanceResponseTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java index 0961d7bff536d..059605457136b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java @@ -337,7 +337,7 @@ public void testToXContent() throws IOException { Map clusterInfo = (Map) json.get("cluster_info"); assertThat( clusterInfo.keySet(), - containsInAnyOrder("nodes", "shard_paths", "shard_sizes", "shard_data_set_sizes", "reserved_sizes", "heap_usage") + containsInAnyOrder("nodes", "shard_paths", "shard_sizes", "shard_data_set_sizes", "reserved_sizes") ); } From 3a1ada2cd5e31d47d249cd8ea05aa94384017e69 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 08:56:25 +1000 Subject: [PATCH 14/30] Only skip disk usage fetches when disk usage is disabled --- .../cluster/InternalClusterInfoService.java | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 551a4ce36c0ca..2fc02da1ea1d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -82,7 +82,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt Property.NodeScope ); - private volatile boolean enabled; + private volatile boolean diskThresholdEnabled; private volatile TimeValue updateFrequency; private volatile TimeValue fetchTimeout; @@ -112,18 +112,18 @@ public InternalClusterInfoService(Settings settings, ClusterService clusterServi this.client = client; this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings); this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings); - this.enabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); + this.diskThresholdEnabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); ClusterSettings clusterSettings = clusterService.getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING, this::setFetchTimeout); clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING, this::setUpdateFrequency); clusterSettings.addSettingsUpdateConsumer( DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, - this::setEnabled + this::setDiskThresholdEnabled ); } - private void setEnabled(boolean enabled) { - this.enabled = enabled; + private void setDiskThresholdEnabled(boolean diskThresholdEnabled) { + this.diskThresholdEnabled = diskThresholdEnabled; } private void setFetchTimeout(TimeValue fetchTimeout) { @@ -182,24 +182,21 @@ private class AsyncRefresh { } void execute() { - if (enabled == false) { - logger.trace("skipping collecting info from cluster, notifying listeners with empty cluster info"); - leastAvailableSpaceUsages = Map.of(); - mostAvailableSpaceUsages = Map.of(); - nodesHeapUsage = Map.of(); - indicesStatsSummary = IndicesStatsSummary.EMPTY; - callListeners(); - return; - } - logger.trace("starting async refresh"); try (var ignoredRefs = fetchRefs) { - try (var ignored = threadPool.getThreadContext().clearTraceContext()) { - fetchNodeStats(); - } - try (var ignored = threadPool.getThreadContext().clearTraceContext()) { - fetchIndicesStats(); + if (diskThresholdEnabled) { + try (var ignored = threadPool.getThreadContext().clearTraceContext()) { + fetchNodeStats(); + } + try (var ignored = threadPool.getThreadContext().clearTraceContext()) { + fetchIndicesStats(); + } + } else { + logger.trace("skipping collecting disk usage info from cluster, notifying listeners with empty cluster info"); + leastAvailableSpaceUsages = Map.of(); + mostAvailableSpaceUsages = Map.of(); + indicesStatsSummary = IndicesStatsSummary.EMPTY; } try (var ignored = threadPool.getThreadContext().clearTraceContext()) { fetchNodesHeapUsage(); From 8fa587fc1bbc6cb64ed7f4195b479d3a6a34db6b Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:09:13 +1000 Subject: [PATCH 15/30] HeapUsage -> ShardHeapUsage --- .../index/shard/IndexShardIT.java | 32 +++++++++--------- .../elasticsearch/cluster/ClusterInfo.java | 18 +++++----- .../cluster/InternalClusterInfoService.java | 22 ++++++------- .../{HeapUsage.java => ShardHeapUsage.java} | 33 ++++++++++--------- ...plier.java => ShardHeapUsageSupplier.java} | 6 ++-- .../routing/allocation/AllocationService.java | 6 ++-- .../cluster/ClusterInfoTests.java | 6 ++-- ...rnalClusterInfoServiceSchedulingTests.java | 12 +++---- 8 files changed, 69 insertions(+), 66 deletions(-) rename server/src/main/java/org/elasticsearch/cluster/{HeapUsage.java => ShardHeapUsage.java} (61%) rename server/src/main/java/org/elasticsearch/cluster/{HeapUsageSupplier.java => ShardHeapUsageSupplier.java} (80%) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index b101ab288d50a..6a479a72e45c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -18,9 +18,9 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.HeapUsage; -import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.InternalClusterInfoService; +import org.elasticsearch.cluster.ShardHeapUsage; +import org.elasticsearch.cluster.ShardHeapUsageSupplier; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -121,7 +121,7 @@ public class IndexShardIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, BogusHeapUsagePlugin.class); + return pluginList(InternalSettingsPlugin.class, BogusShardHeapUsagePlugin.class); } public void testLockTryingToDelete() throws Exception { @@ -263,13 +263,13 @@ public void testHeapUsageEstimateIsPresent() { InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); ClusterInfoServiceUtils.refresh(clusterInfoService); ClusterState state = getInstanceFromNode(ClusterService.class).state(); - Map heapUsages = clusterInfoService.getClusterInfo().getNodesHeapUsage(); - assertNotNull(heapUsages); - assertEquals(state.nodes().size(), heapUsages.size()); + Map shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages(); + assertNotNull(shardHeapUsages); + assertEquals(state.nodes().size(), shardHeapUsages.size()); for (DiscoveryNode node : state.nodes()) { - assertTrue(heapUsages.containsKey(node.getId())); - HeapUsage heapUsage = heapUsages.get(node.getId()); - assertThat(heapUsage.freeBytes(), lessThanOrEqualTo(heapUsage.totalBytes())); + assertTrue(shardHeapUsages.containsKey(node.getId())); + ShardHeapUsage shardHeapUsage = shardHeapUsages.get(node.getId()); + assertThat(shardHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(shardHeapUsage.totalBytes())); } } @@ -816,34 +816,34 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable> listener) { + public void getClusterHeapUsage(ActionListener> listener) { ActionListener.completeWith( listener, () -> clusterService.state().nodes().stream().collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { final long maxHeap = randomNonNegativeLong(); final long usedHeap = (long) (randomFloat() * maxHeap); - return new HeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); + return new ShardHeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); })) ); } } - public static class BogusHeapUsagePlugin extends Plugin implements ClusterPlugin { + public static class BogusShardHeapUsagePlugin extends Plugin implements ClusterPlugin { - public BogusHeapUsagePlugin() {} + public BogusShardHeapUsagePlugin() {} @Override public Collection createComponents(PluginServices services) { - BogusHeapUsageSupplier bogusHeapUsageSupplier = new BogusHeapUsageSupplier(services.clusterService()); + BogusShardShardHeapUsageSupplier bogusHeapUsageSupplier = new BogusShardShardHeapUsageSupplier(services.clusterService()); services.allocationService().setHeapUsageSupplier(bogusHeapUsageSupplier); return List.of(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index fc981d97e7efc..54b1cb354e0bf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -57,7 +57,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { final Map shardDataSetSizes; final Map dataPath; final Map reservedSpace; - final Map nodesHeapUsage; + final Map shardHeapUsages; protected ClusterInfo() { this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); @@ -72,7 +72,7 @@ protected ClusterInfo() { * @param shardDataSetSizes a shard id to data set size in bytes mapping per shard * @param dataPath the shard routing to datapath mapping * @param reservedSpace reserved space per shard broken down by node and data path - * @param nodesHeapUsage heap usage broken down by node + * @param shardHeapUsages shard heap usage broken down by node * @see #shardIdentifierFromRouting */ public ClusterInfo( @@ -82,7 +82,7 @@ public ClusterInfo( Map shardDataSetSizes, Map dataPath, Map reservedSpace, - Map nodesHeapUsage + Map shardHeapUsages ) { this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage); this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage); @@ -90,7 +90,7 @@ public ClusterInfo( this.shardDataSetSizes = Map.copyOf(shardDataSetSizes); this.dataPath = Map.copyOf(dataPath); this.reservedSpace = Map.copyOf(reservedSpace); - this.nodesHeapUsage = Map.copyOf(nodesHeapUsage); + this.shardHeapUsages = Map.copyOf(shardHeapUsages); } public ClusterInfo(StreamInput in) throws IOException { @@ -103,9 +103,9 @@ public ClusterInfo(StreamInput in) throws IOException { : in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString); this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new); if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { - this.nodesHeapUsage = in.readImmutableMap(HeapUsage::new); + this.shardHeapUsages = in.readImmutableMap(ShardHeapUsage::new); } else { - this.nodesHeapUsage = Map.of(); + this.shardHeapUsages = Map.of(); } } @@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeMap(this.reservedSpace); if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { - out.writeMap(this.nodesHeapUsage, StreamOutput::writeWriteable); + out.writeMap(this.shardHeapUsages, StreamOutput::writeWriteable); } } @@ -214,8 +214,8 @@ public Iterator toXContentChunked(ToXContent.Params params * Also note that the map may not be complete, it may contain none, or a subset of the nodes in * the cluster at any time. It may also contain entries for nodes that have since left the cluster. */ - public Map getNodesHeapUsage() { - return nodesHeapUsage; + public Map getShardHeapUsages() { + return shardHeapUsages; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 2fc02da1ea1d4..de2b9a0f0e4e8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -89,7 +89,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; private volatile IndicesStatsSummary indicesStatsSummary; - private volatile Map nodesHeapUsage; + private volatile Map shardHeapUsages; private final ThreadPool threadPool; private final Client client; @@ -98,7 +98,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); - private HeapUsageSupplier heapUsageSupplier = HeapUsageSupplier.EMPTY; + private ShardHeapUsageSupplier shardHeapUsageSupplier = ShardHeapUsageSupplier.EMPTY; private AsyncRefresh currentRefresh; private RefreshScheduler refreshScheduler; @@ -106,7 +106,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); - this.nodesHeapUsage = Map.of(); + this.shardHeapUsages = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; @@ -137,11 +137,11 @@ void setUpdateFrequency(TimeValue updateFrequency) { /** * This can be provided by plugins, which are initialised long after the ClusterInfoService is created * - * @param heapUsageSupplier The HeapUsageSupplier to use + * @param shardHeapUsageSupplier The HeapUsageSupplier to use */ - public void setHeapUsageSupplier(HeapUsageSupplier heapUsageSupplier) { - assert this.heapUsageSupplier == HeapUsageSupplier.EMPTY; - this.heapUsageSupplier = heapUsageSupplier; + public void setShardHeapUsageSupplier(ShardHeapUsageSupplier shardHeapUsageSupplier) { + assert this.shardHeapUsageSupplier == ShardHeapUsageSupplier.EMPTY; + this.shardHeapUsageSupplier = shardHeapUsageSupplier; } @Override @@ -205,10 +205,10 @@ void execute() { } private void fetchNodesHeapUsage() { - heapUsageSupplier.getClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { + shardHeapUsageSupplier.getClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { @Override - public void onResponse(Map stringHeapUsageMap) { - nodesHeapUsage = stringHeapUsageMap; + public void onResponse(Map stringHeapUsageMap) { + shardHeapUsages = stringHeapUsageMap; } @Override @@ -442,7 +442,7 @@ public ClusterInfo getClusterInfo() { indicesStatsSummary.shardDataSetSizes, indicesStatsSummary.dataPath, indicesStatsSummary.reservedSpace, - nodesHeapUsage + shardHeapUsages ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java similarity index 61% rename from server/src/main/java/org/elasticsearch/cluster/HeapUsage.java rename to server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index f4bdc77268cfe..80d1a6fc46c7e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/HeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -19,15 +19,18 @@ import java.io.IOException; /** - * Record representing the heap usage for a single cluster node + * Record representing an estimate of the heap used by allocated shards and ongoing merges on a particular node */ -public record HeapUsage(String nodeId, String nodeName, long totalBytes, long usedBytes) implements ToXContentFragment, Writeable { +public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, long estimatedUsageBytes) + implements + ToXContentFragment, + Writeable { - public HeapUsage { - assert usedBytes <= totalBytes; + public ShardHeapUsage { + assert estimatedUsageBytes <= totalBytes; } - public HeapUsage(StreamInput in) throws IOException { + public ShardHeapUsage(StreamInput in) throws IOException { this(in.readString(), in.readString(), in.readVLong(), in.readVLong()); } @@ -36,16 +39,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(this.nodeId); out.writeString(this.nodeName); out.writeVLong(this.totalBytes); - out.writeVLong(this.usedBytes); + out.writeVLong(this.estimatedUsageBytes); } public XContentBuilder toShortXContent(XContentBuilder builder) throws IOException { builder.field("node_name", this.nodeName); builder.humanReadableField("total_heap_bytes", "total", ByteSizeValue.ofBytes(this.totalBytes)); - builder.humanReadableField("used_heap_bytes", "used", ByteSizeValue.ofBytes(this.usedBytes)); - builder.humanReadableField("free_heap_bytes", "free", ByteSizeValue.ofBytes(this.freeBytes())); - builder.field("free_heap_percent", truncatePercent(this.freeHeapAsPercentage())); - builder.field("used_heap_percent", truncatePercent(this.usedHeapAsPercentage())); + builder.humanReadableField("estimated_usage_bytes", "used", ByteSizeValue.ofBytes(this.estimatedUsageBytes)); + builder.humanReadableField("estimated_free_bytes", "free", ByteSizeValue.ofBytes(this.estimatedFreeBytes())); + builder.field("estimated_free_percent", truncatePercent(this.freeHeapAsPercentage())); + builder.field("estimated_usage_percent", truncatePercent(this.estimatedUsageAsPercentage())); return builder; } @@ -57,15 +60,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public double freeHeapAsPercentage() { - return 100.0 - usedHeapAsPercentage(); + return 100.0 - estimatedUsageAsPercentage(); } - public double usedHeapAsPercentage() { - return 100.0 * usedBytes / (double) totalBytes; + public double estimatedUsageAsPercentage() { + return 100.0 * estimatedUsageBytes / (double) totalBytes; } - public long freeBytes() { - return totalBytes - usedBytes; + public long estimatedFreeBytes() { + return totalBytes - estimatedUsageBytes; } private static double truncatePercent(double pct) { diff --git a/server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java similarity index 80% rename from server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java rename to server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java index 54bca56d376b2..6de41a2d676bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/HeapUsageSupplier.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java @@ -13,17 +13,17 @@ import java.util.Map; -public interface HeapUsageSupplier { +public interface ShardHeapUsageSupplier { /** * This will be used when there are no heap usage suppliers available */ - HeapUsageSupplier EMPTY = listener -> listener.onResponse(Map.of()); + ShardHeapUsageSupplier EMPTY = listener -> listener.onResponse(Map.of()); /** * Get the heap usage for every node in the cluster * * @param listener The listener which will receive the results */ - void getClusterHeapUsage(ActionListener> listener); + void getClusterHeapUsage(ActionListener> listener); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index b685896933d0f..2797e60e8aff5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -15,9 +15,9 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.HeapUsageSupplier; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.ShardHeapUsageSupplier; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -639,9 +639,9 @@ public void addAllocFailuresResetListenerTo(ClusterService clusterService) { }); } - public void setHeapUsageSupplier(HeapUsageSupplier heapUsageSupplier) { + public void setHeapUsageSupplier(ShardHeapUsageSupplier shardHeapUsageSupplier) { if (clusterInfoService instanceof InternalClusterInfoService icis) { - icis.setHeapUsageSupplier(heapUsageSupplier); + icis.setShardHeapUsageSupplier(shardHeapUsageSupplier); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java index 28cbd89294c30..775b824250a08 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java @@ -46,13 +46,13 @@ public static ClusterInfo randomClusterInfo() { ); } - private static Map randomNodeHeapUsage() { + private static Map randomNodeHeapUsage() { int numEntries = randomIntBetween(0, 128); - Map nodeHeapUsage = new HashMap<>(numEntries); + Map nodeHeapUsage = new HashMap<>(numEntries); for (int i = 0; i < numEntries; i++) { String key = randomAlphaOfLength(32); final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE); - final HeapUsage diskUsage = new HeapUsage( + final ShardHeapUsage diskUsage = new ShardHeapUsage( randomAlphaOfLength(4), randomAlphaOfLength(4), totalBytes, diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index 24593e3a7c602..2fc5a349ef1c6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -77,8 +77,8 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService(settings, clusterService, threadPool, client); - final HeapUsageSupplier mockHeapUsageSupplier = spy(new StubHeapUsageSupplier()); - clusterInfoService.setHeapUsageSupplier(mockHeapUsageSupplier); + final ShardHeapUsageSupplier mockShardHeapUsageSupplier = spy(new StubShardShardHeapUsageSupplier()); + clusterInfoService.setShardHeapUsageSupplier(mockShardHeapUsageSupplier); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); @@ -114,13 +114,13 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { deterministicTaskQueue.runAllRunnableTasks(); for (int i = 0; i < 3; i++) { - Mockito.clearInvocations(mockHeapUsageSupplier); + Mockito.clearInvocations(mockShardHeapUsageSupplier); final int initialRequestCount = client.requestCount; final long duration = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings).millis(); runFor(deterministicTaskQueue, duration); deterministicTaskQueue.runAllRunnableTasks(); assertThat(client.requestCount, equalTo(initialRequestCount + 2)); // should have run two client requests per interval - verify(mockHeapUsageSupplier).getClusterHeapUsage(any()); // Should poll for heap usage once per interval + verify(mockShardHeapUsageSupplier).getClusterHeapUsage(any()); // Should poll for heap usage once per interval } final AtomicBoolean failMaster2 = new AtomicBoolean(); @@ -137,10 +137,10 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { assertFalse(deterministicTaskQueue.hasDeferredTasks()); } - private static class StubHeapUsageSupplier implements HeapUsageSupplier { + private static class StubShardShardHeapUsageSupplier implements ShardHeapUsageSupplier { @Override - public void getClusterHeapUsage(ActionListener> listener) { + public void getClusterHeapUsage(ActionListener> listener) { listener.onResponse(Map.of()); } } From 2c42a82d0e3db550456f55af730f89a10d25e953 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:17:07 +1000 Subject: [PATCH 16/30] icis -> internalClusterInfoService --- .../cluster/routing/allocation/AllocationService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 2797e60e8aff5..974d68f68c55a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -640,8 +640,8 @@ public void addAllocFailuresResetListenerTo(ClusterService clusterService) { } public void setHeapUsageSupplier(ShardHeapUsageSupplier shardHeapUsageSupplier) { - if (clusterInfoService instanceof InternalClusterInfoService icis) { - icis.setShardHeapUsageSupplier(shardHeapUsageSupplier); + if (clusterInfoService instanceof InternalClusterInfoService internalClusterInfoService) { + internalClusterInfoService.setShardHeapUsageSupplier(shardHeapUsageSupplier); } } From 58402bd42dd11f819dba46c1caaa81662dde4dc5 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:18:00 +1000 Subject: [PATCH 17/30] diskUsage -> shardHeapUsage --- .../test/java/org/elasticsearch/cluster/ClusterInfoTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java index 775b824250a08..5de885dabc674 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java @@ -52,13 +52,13 @@ private static Map randomNodeHeapUsage() { for (int i = 0; i < numEntries; i++) { String key = randomAlphaOfLength(32); final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE); - final ShardHeapUsage diskUsage = new ShardHeapUsage( + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage( randomAlphaOfLength(4), randomAlphaOfLength(4), totalBytes, randomIntBetween(0, totalBytes) ); - nodeHeapUsage.put(key, diskUsage); + nodeHeapUsage.put(key, shardHeapUsage); } return nodeHeapUsage; } From 63bbea8ee6805e4eef4e7c4c24d4072f9e38eb89 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:19:58 +1000 Subject: [PATCH 18/30] Note about not serializing shardHeapUsages --- server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index 54b1cb354e0bf..460ed5e119c1a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -204,6 +204,8 @@ public Iterator toXContentChunked(ToXContent.Params params return builder.endObject(); // NodeAndPath }), endArray() // end "reserved_sizes" + // NOTE: We don't serialize shardHeapUsages at this stage, to avoid + // committing to API payloads until the feature is settled ); } From 0cacdc7bac571e252c7625d7444b59d6a8f8bdaa Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:21:31 +1000 Subject: [PATCH 19/30] Remove unused serialization interface/methods --- .../elasticsearch/cluster/ShardHeapUsage.java | 37 +------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index 80d1a6fc46c7e..caaef11eb0106 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -12,19 +12,13 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; /** * Record representing an estimate of the heap used by allocated shards and ongoing merges on a particular node */ -public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, long estimatedUsageBytes) - implements - ToXContentFragment, - Writeable { +public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, long estimatedUsageBytes) implements Writeable { public ShardHeapUsage { assert estimatedUsageBytes <= totalBytes; @@ -42,36 +36,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(this.estimatedUsageBytes); } - public XContentBuilder toShortXContent(XContentBuilder builder) throws IOException { - builder.field("node_name", this.nodeName); - builder.humanReadableField("total_heap_bytes", "total", ByteSizeValue.ofBytes(this.totalBytes)); - builder.humanReadableField("estimated_usage_bytes", "used", ByteSizeValue.ofBytes(this.estimatedUsageBytes)); - builder.humanReadableField("estimated_free_bytes", "free", ByteSizeValue.ofBytes(this.estimatedFreeBytes())); - builder.field("estimated_free_percent", truncatePercent(this.freeHeapAsPercentage())); - builder.field("estimated_usage_percent", truncatePercent(this.estimatedUsageAsPercentage())); - return builder; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("node_id", this.nodeId); - toShortXContent(builder); - return builder; - } - - public double freeHeapAsPercentage() { - return 100.0 - estimatedUsageAsPercentage(); - } - - public double estimatedUsageAsPercentage() { - return 100.0 * estimatedUsageBytes / (double) totalBytes; - } - public long estimatedFreeBytes() { return totalBytes - estimatedUsageBytes; } - - private static double truncatePercent(double pct) { - return Math.round(pct * 10.0) / 10.0; - } } From dd73d37c2623acb6ca7d5233a7041148a7e170eb Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:22:13 +1000 Subject: [PATCH 20/30] Additional assertions --- .../src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index caaef11eb0106..1030d4a968b74 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -22,6 +22,8 @@ public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, lo public ShardHeapUsage { assert estimatedUsageBytes <= totalBytes; + assert totalBytes >= 0; + assert estimatedUsageBytes >= 0; } public ShardHeapUsage(StreamInput in) throws IOException { From 765ade8acb08f6624c9c81d3afb0c1262963b955 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:23:08 +1000 Subject: [PATCH 21/30] Clear shardHeapUsages on failure to fetch --- .../org/elasticsearch/cluster/InternalClusterInfoService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index de2b9a0f0e4e8..b95712aa4cabc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -214,6 +214,7 @@ public void onResponse(Map stringHeapUsageMap) { @Override public void onFailure(Exception e) { logger.warn("failed to fetch heap usage for nodes", e); + shardHeapUsages = Map.of(); } }, fetchRefs.acquire())); } From e26b62fcd76ab8ab03d11c7f3cb15cf34e7c72bf Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 09:24:00 +1000 Subject: [PATCH 22/30] Fix naming --- .../org/elasticsearch/cluster/InternalClusterInfoService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index b95712aa4cabc..7f02ff6e640a4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -207,8 +207,8 @@ void execute() { private void fetchNodesHeapUsage() { shardHeapUsageSupplier.getClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { @Override - public void onResponse(Map stringHeapUsageMap) { - shardHeapUsages = stringHeapUsageMap; + public void onResponse(Map currentShardHeapUsages) { + shardHeapUsages = currentShardHeapUsages; } @Override From 55637b668de8e45dc1d0f5079010d2ba438505e1 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 12:00:08 +1000 Subject: [PATCH 23/30] Restore + test percentage methods --- .../elasticsearch/cluster/ShardHeapUsage.java | 8 ++++ .../cluster/ShardHeapUsageTests.java | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index 1030d4a968b74..3247ec13c8b88 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -41,4 +41,12 @@ public void writeTo(StreamOutput out) throws IOException { public long estimatedFreeBytes() { return totalBytes - estimatedUsageBytes; } + + public double estimatedFreeBytesAsPercentage() { + return 100.0 - estimatedUsageAsPercentage(); + } + + public double estimatedUsageAsPercentage() { + return 100.0 * estimatedUsageBytes / (double) totalBytes; + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java new file mode 100644 index 0000000000000..fd557b00e0503 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class ShardHeapUsageTests extends ESTestCase { + + public void testEstimatedUsageAsPercentage() { + final long totalBytes = randomNonNegativeLong(); + final long estimatedUsageBytes = randomLongBetween(0, totalBytes); + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), randomIdentifier(), totalBytes, estimatedUsageBytes); + assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), greaterThanOrEqualTo(0.0)); + assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), lessThanOrEqualTo(100.0)); + assertEquals(shardHeapUsage.estimatedUsageAsPercentage(), 100.0 * estimatedUsageBytes / totalBytes, 0.0001); + } + + public void testEstimatedFreeBytesAsPercentage() { + final long totalBytes = randomNonNegativeLong(); + final long estimatedUsageBytes = randomLongBetween(0, totalBytes); + final long estimatedFreeBytes = totalBytes - estimatedUsageBytes; + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), randomIdentifier(), totalBytes, estimatedUsageBytes); + assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), greaterThanOrEqualTo(0.0)); + assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), lessThanOrEqualTo(100.0)); + assertEquals(shardHeapUsage.estimatedFreeBytesAsPercentage(), 100.0 * estimatedFreeBytes / totalBytes, 0.0001); + } +} From f4b90b5a8277bc641fe9860f690a7828ab89c41a Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 16:12:24 +1000 Subject: [PATCH 24/30] Load ShardHeapUsageSupplier via SPI --- .../index/shard/IndexShardIT.java | 32 ++++++++++++------- .../cluster/InternalClusterInfoService.java | 21 ++++++------ .../routing/allocation/AllocationService.java | 8 ----- .../node/NodeServiceProvider.java | 13 +++++++- ...rnalClusterInfoServiceSchedulingTests.java | 9 ++++-- ...asticsearch.cluster.ShardHeapUsageSupplier | 10 ++++++ .../MockInternalClusterInfoService.java | 2 +- 7 files changed, 59 insertions(+), 36 deletions(-) create mode 100644 server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 6a479a72e45c0..f0fd9128843fb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; @@ -816,36 +817,43 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable> listener) { ActionListener.completeWith( listener, - () -> clusterService.state().nodes().stream().collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { - final long maxHeap = randomNonNegativeLong(); - final long usedHeap = (long) (randomFloat() * maxHeap); - return new ShardHeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); - })) + () -> plugin.getClusterService() + .state() + .nodes() + .stream() + .collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { + final long maxHeap = randomNonNegativeLong(); + final long usedHeap = (long) (randomFloat() * maxHeap); + return new ShardHeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); + })) ); } } public static class BogusShardHeapUsagePlugin extends Plugin implements ClusterPlugin { - public BogusShardHeapUsagePlugin() {} + private final SetOnce clusterService = new SetOnce<>(); @Override public Collection createComponents(PluginServices services) { - BogusShardShardHeapUsageSupplier bogusHeapUsageSupplier = new BogusShardShardHeapUsageSupplier(services.clusterService()); - services.allocationService().setHeapUsageSupplier(bogusHeapUsageSupplier); + clusterService.set(services.clusterService()); return List.of(); } + + public ClusterService getClusterService() { + return clusterService.get(); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 7f02ff6e640a4..f5bef2c81d44f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -97,19 +97,26 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); + private final ShardHeapUsageSupplier shardHeapUsageSupplier; - private ShardHeapUsageSupplier shardHeapUsageSupplier = ShardHeapUsageSupplier.EMPTY; private AsyncRefresh currentRefresh; private RefreshScheduler refreshScheduler; @SuppressWarnings("this-escape") - public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { + public InternalClusterInfoService( + Settings settings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + ShardHeapUsageSupplier shardHeapUsageSupplier + ) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); this.shardHeapUsages = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; + this.shardHeapUsageSupplier = shardHeapUsageSupplier; this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings); this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings); this.diskThresholdEnabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); @@ -134,16 +141,6 @@ void setUpdateFrequency(TimeValue updateFrequency) { this.updateFrequency = updateFrequency; } - /** - * This can be provided by plugins, which are initialised long after the ClusterInfoService is created - * - * @param shardHeapUsageSupplier The HeapUsageSupplier to use - */ - public void setShardHeapUsageSupplier(ShardHeapUsageSupplier shardHeapUsageSupplier) { - assert this.shardHeapUsageSupplier == ShardHeapUsageSupplier.EMPTY; - this.shardHeapUsageSupplier = shardHeapUsageSupplier; - } - @Override public void clusterChanged(ClusterChangedEvent event) { final Runnable newRefresh; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 974d68f68c55a..263c2362c9ed6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -15,9 +15,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.ShardHeapUsageSupplier; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -639,12 +637,6 @@ public void addAllocFailuresResetListenerTo(ClusterService clusterService) { }); } - public void setHeapUsageSupplier(ShardHeapUsageSupplier shardHeapUsageSupplier) { - if (clusterInfoService instanceof InternalClusterInfoService internalClusterInfoService) { - internalClusterInfoService.setShardHeapUsageSupplier(shardHeapUsageSupplier); - } - } - /** * We should reset allocation/relocation failure count to allow further retries when: * diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 1c3f19b20b4dc..4acc5bbb4f6c8 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.InternalClusterInfoService; +import org.elasticsearch.cluster.ShardHeapUsageSupplier; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -74,7 +75,17 @@ ClusterInfoService newClusterInfoService( ThreadPool threadPool, NodeClient client ) { - final InternalClusterInfoService service = new InternalClusterInfoService(settings, clusterService, threadPool, client); + final ShardHeapUsageSupplier shardHeapUsageSupplier = pluginsService.loadSingletonServiceProvider( + ShardHeapUsageSupplier.class, + () -> ShardHeapUsageSupplier.EMPTY + ); + final InternalClusterInfoService service = new InternalClusterInfoService( + settings, + clusterService, + threadPool, + client, + shardHeapUsageSupplier + ); if (DiscoveryNode.isMasterNode(settings)) { // listen for state changes (this node starts/stops being the elected master, or new nodes are added) clusterService.addListener(service); diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index 2fc5a349ef1c6..e2407a560d157 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -76,9 +76,14 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final ClusterService clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService); final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); - final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService(settings, clusterService, threadPool, client); final ShardHeapUsageSupplier mockShardHeapUsageSupplier = spy(new StubShardShardHeapUsageSupplier()); - clusterInfoService.setShardHeapUsageSupplier(mockShardHeapUsageSupplier); + final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService( + settings, + clusterService, + threadPool, + client, + mockShardHeapUsageSupplier + ); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); diff --git a/server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier b/server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier new file mode 100644 index 0000000000000..953100ac7324f --- /dev/null +++ b/server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier @@ -0,0 +1,10 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". +# + +org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageSupplier diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 49a36f64e2811..b888cbfa6a21f 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -43,7 +43,7 @@ public static class TestPlugin extends Plugin {} private volatile BiFunction diskUsageFunction; public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) { - super(settings, clusterService, threadPool, client); + super(settings, clusterService, threadPool, client, ShardHeapUsageSupplier.EMPTY); } public void setDiskUsageFunctionAndRefresh(BiFunction diskUsageFn) { From 0789fefa98a5e506e27d0b5973676e78839a739c Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Thu, 5 Jun 2025 22:09:44 +1000 Subject: [PATCH 25/30] Move SPI config to internalClusterTest --- .../services/org.elasticsearch.cluster.ShardHeapUsageSupplier | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/{test => internalClusterTest}/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier (100%) diff --git a/server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier b/server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier similarity index 100% rename from server/src/test/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier rename to server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier From 7b5bf95d37849d3c93cf69b6594c4629057f7130 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Sat, 7 Jun 2025 10:17:22 +1000 Subject: [PATCH 26/30] *Supplier -> *Collector --- .../org/elasticsearch/index/shard/IndexShardIT.java | 8 ++++---- ...rg.elasticsearch.cluster.ShardHeapUsageCollector} | 2 +- .../cluster/InternalClusterInfoService.java | 8 ++++---- ...ageSupplier.java => ShardHeapUsageCollector.java} | 8 ++++---- .../org/elasticsearch/node/NodeServiceProvider.java | 10 +++++----- .../InternalClusterInfoServiceSchedulingTests.java | 12 ++++++------ .../cluster/MockInternalClusterInfoService.java | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) rename server/src/internalClusterTest/resources/META-INF/services/{org.elasticsearch.cluster.ShardHeapUsageSupplier => org.elasticsearch.cluster.ShardHeapUsageCollector} (97%) rename server/src/main/java/org/elasticsearch/cluster/{ShardHeapUsageSupplier.java => ShardHeapUsageCollector.java} (73%) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index f0fd9128843fb..96b054b3392f6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -21,7 +21,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.ShardHeapUsage; -import org.elasticsearch.cluster.ShardHeapUsageSupplier; +import org.elasticsearch.cluster.ShardHeapUsageCollector; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -817,16 +817,16 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable> listener) { + public void collectClusterHeapUsage(ActionListener> listener) { ActionListener.completeWith( listener, () -> plugin.getClusterService() diff --git a/server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier b/server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector similarity index 97% rename from server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier rename to server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector index 953100ac7324f..15b62c8240f25 100644 --- a/server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageSupplier +++ b/server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector @@ -7,4 +7,4 @@ # License v3.0 only", or the "Server Side Public License, v 1". # -org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageSupplier +org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageCollector diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index f5bef2c81d44f..35cbac38680f4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -97,7 +97,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private final Object mutex = new Object(); private final List> nextRefreshListeners = new ArrayList<>(); - private final ShardHeapUsageSupplier shardHeapUsageSupplier; + private final ShardHeapUsageCollector shardHeapUsageCollector; private AsyncRefresh currentRefresh; private RefreshScheduler refreshScheduler; @@ -108,7 +108,7 @@ public InternalClusterInfoService( ClusterService clusterService, ThreadPool threadPool, Client client, - ShardHeapUsageSupplier shardHeapUsageSupplier + ShardHeapUsageCollector shardHeapUsageCollector ) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); @@ -116,7 +116,7 @@ public InternalClusterInfoService( this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; - this.shardHeapUsageSupplier = shardHeapUsageSupplier; + this.shardHeapUsageCollector = shardHeapUsageCollector; this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings); this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings); this.diskThresholdEnabled = DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); @@ -202,7 +202,7 @@ void execute() { } private void fetchNodesHeapUsage() { - shardHeapUsageSupplier.getClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { + shardHeapUsageCollector.collectClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(Map currentShardHeapUsages) { shardHeapUsages = currentShardHeapUsages; diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java similarity index 73% rename from server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java rename to server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java index 6de41a2d676bd..2218300d13f1d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageSupplier.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java @@ -13,17 +13,17 @@ import java.util.Map; -public interface ShardHeapUsageSupplier { +public interface ShardHeapUsageCollector { /** * This will be used when there are no heap usage suppliers available */ - ShardHeapUsageSupplier EMPTY = listener -> listener.onResponse(Map.of()); + ShardHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of()); /** - * Get the heap usage for every node in the cluster + * Collect the heap usage for every node in the cluster * * @param listener The listener which will receive the results */ - void getClusterHeapUsage(ActionListener> listener); + void collectClusterHeapUsage(ActionListener> listener); } diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 4acc5bbb4f6c8..8ef81eb35543a 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -13,7 +13,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.InternalClusterInfoService; -import org.elasticsearch.cluster.ShardHeapUsageSupplier; +import org.elasticsearch.cluster.ShardHeapUsageCollector; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -75,16 +75,16 @@ ClusterInfoService newClusterInfoService( ThreadPool threadPool, NodeClient client ) { - final ShardHeapUsageSupplier shardHeapUsageSupplier = pluginsService.loadSingletonServiceProvider( - ShardHeapUsageSupplier.class, - () -> ShardHeapUsageSupplier.EMPTY + final ShardHeapUsageCollector shardHeapUsageCollector = pluginsService.loadSingletonServiceProvider( + ShardHeapUsageCollector.class, + () -> ShardHeapUsageCollector.EMPTY ); final InternalClusterInfoService service = new InternalClusterInfoService( settings, clusterService, threadPool, client, - shardHeapUsageSupplier + shardHeapUsageCollector ); if (DiscoveryNode.isMasterNode(settings)) { // listen for state changes (this node starts/stops being the elected master, or new nodes are added) diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index e2407a560d157..17721c2454e87 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -76,13 +76,13 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { final ClusterService clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService); final FakeClusterInfoServiceClient client = new FakeClusterInfoServiceClient(threadPool); - final ShardHeapUsageSupplier mockShardHeapUsageSupplier = spy(new StubShardShardHeapUsageSupplier()); + final ShardHeapUsageCollector mockShardHeapUsageCollector = spy(new StubShardShardHeapUsageCollector()); final InternalClusterInfoService clusterInfoService = new InternalClusterInfoService( settings, clusterService, threadPool, client, - mockShardHeapUsageSupplier + mockShardHeapUsageCollector ); clusterService.addListener(clusterInfoService); clusterInfoService.addListener(ignored -> {}); @@ -119,13 +119,13 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { deterministicTaskQueue.runAllRunnableTasks(); for (int i = 0; i < 3; i++) { - Mockito.clearInvocations(mockShardHeapUsageSupplier); + Mockito.clearInvocations(mockShardHeapUsageCollector); final int initialRequestCount = client.requestCount; final long duration = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings).millis(); runFor(deterministicTaskQueue, duration); deterministicTaskQueue.runAllRunnableTasks(); assertThat(client.requestCount, equalTo(initialRequestCount + 2)); // should have run two client requests per interval - verify(mockShardHeapUsageSupplier).getClusterHeapUsage(any()); // Should poll for heap usage once per interval + verify(mockShardHeapUsageCollector).collectClusterHeapUsage(any()); // Should poll for heap usage once per interval } final AtomicBoolean failMaster2 = new AtomicBoolean(); @@ -142,10 +142,10 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { assertFalse(deterministicTaskQueue.hasDeferredTasks()); } - private static class StubShardShardHeapUsageSupplier implements ShardHeapUsageSupplier { + private static class StubShardShardHeapUsageCollector implements ShardHeapUsageCollector { @Override - public void getClusterHeapUsage(ActionListener> listener) { + public void collectClusterHeapUsage(ActionListener> listener) { listener.onResponse(Map.of()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index b888cbfa6a21f..64cd81e1a9ab2 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -43,7 +43,7 @@ public static class TestPlugin extends Plugin {} private volatile BiFunction diskUsageFunction; public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) { - super(settings, clusterService, threadPool, client, ShardHeapUsageSupplier.EMPTY); + super(settings, clusterService, threadPool, client, ShardHeapUsageCollector.EMPTY); } public void setDiskUsageFunctionAndRefresh(BiFunction diskUsageFn) { From cd6b7e93f159861eed4ea8bfefa36f964b4b2ef4 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 10 Jun 2025 15:25:43 +1000 Subject: [PATCH 27/30] Don't assert estimate <= max heap --- .../src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index 3247ec13c8b88..fb2b951ec92b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -21,7 +21,6 @@ public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, long estimatedUsageBytes) implements Writeable { public ShardHeapUsage { - assert estimatedUsageBytes <= totalBytes; assert totalBytes >= 0; assert estimatedUsageBytes >= 0; } From d831194506a747e97f151da2cb691aeef737fd7f Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 11 Jun 2025 12:25:19 +1000 Subject: [PATCH 28/30] Use node stats to retrieve max heap size --- .../index/shard/IndexShardIT.java | 8 +--- .../cluster/InternalClusterInfoService.java | 40 +++++++++++++------ .../elasticsearch/cluster/ShardHeapUsage.java | 5 +-- .../cluster/ShardHeapUsageCollector.java | 13 ++++-- .../cluster/ClusterInfoTests.java | 1 - ...rnalClusterInfoServiceSchedulingTests.java | 2 +- .../cluster/ShardHeapUsageTests.java | 4 +- 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 96b054b3392f6..d7c9c00d82c87 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -826,18 +826,14 @@ public BogusShardShardHeapUsageCollector(BogusShardHeapUsagePlugin plugin) { } @Override - public void collectClusterHeapUsage(ActionListener> listener) { + public void collectClusterHeapUsage(ActionListener> listener) { ActionListener.completeWith( listener, () -> plugin.getClusterService() .state() .nodes() .stream() - .collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> { - final long maxHeap = randomNonNegativeLong(); - final long usedHeap = (long) (randomFloat() * maxHeap); - return new ShardHeapUsage(node.getId(), node.getName(), maxHeap, usedHeap); - })) + .collect(Collectors.toUnmodifiableMap(DiscoveryNode::getId, node -> randomNonNegativeLong())) ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 35cbac38680f4..43c3730196b91 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; @@ -88,8 +89,9 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; + private volatile Map maxHeapPerNode; + private volatile Map estimatedHeapUsagePerNode; private volatile IndicesStatsSummary indicesStatsSummary; - private volatile Map shardHeapUsages; private final ThreadPool threadPool; private final Client client; @@ -183,18 +185,16 @@ void execute() { try (var ignoredRefs = fetchRefs) { if (diskThresholdEnabled) { - try (var ignored = threadPool.getThreadContext().clearTraceContext()) { - fetchNodeStats(); - } try (var ignored = threadPool.getThreadContext().clearTraceContext()) { fetchIndicesStats(); } } else { logger.trace("skipping collecting disk usage info from cluster, notifying listeners with empty cluster info"); - leastAvailableSpaceUsages = Map.of(); - mostAvailableSpaceUsages = Map.of(); indicesStatsSummary = IndicesStatsSummary.EMPTY; } + try (var ignored = threadPool.getThreadContext().clearTraceContext()) { + fetchNodeStats(); + } try (var ignored = threadPool.getThreadContext().clearTraceContext()) { fetchNodesHeapUsage(); } @@ -204,14 +204,14 @@ void execute() { private void fetchNodesHeapUsage() { shardHeapUsageCollector.collectClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { @Override - public void onResponse(Map currentShardHeapUsages) { - shardHeapUsages = currentShardHeapUsages; + public void onResponse(Map currentShardHeapUsages) { + estimatedHeapUsagePerNode = currentShardHeapUsages; } @Override public void onFailure(Exception e) { logger.warn("failed to fetch heap usage for nodes", e); - shardHeapUsages = Map.of(); + estimatedHeapUsagePerNode = Map.of(); } }, fetchRefs.acquire())); } @@ -311,6 +311,7 @@ private void fetchNodeStats() { nodesStatsRequest.setIncludeShardsStats(false); nodesStatsRequest.clear(); nodesStatsRequest.addMetric(NodesStatsRequestParameters.Metric.FS); + nodesStatsRequest.addMetric(NodesStatsRequestParameters.Metric.JVM); nodesStatsRequest.setTimeout(fetchTimeout); client.admin().cluster().nodesStats(nodesStatsRequest, ActionListener.releaseAfter(new ActionListener<>() { @Override @@ -323,13 +324,16 @@ public void onResponse(NodesStatsResponse nodesStatsResponse) { Map leastAvailableUsagesBuilder = new HashMap<>(); Map mostAvailableUsagesBuilder = new HashMap<>(); - fillDiskUsagePerNode( + Map maxHeapPerNodeBuilder = new HashMap<>(); + processNodeStatsArray( adjustNodesStats(nodesStatsResponse.getNodes()), leastAvailableUsagesBuilder, - mostAvailableUsagesBuilder + mostAvailableUsagesBuilder, + maxHeapPerNodeBuilder ); leastAvailableSpaceUsages = Map.copyOf(leastAvailableUsagesBuilder); mostAvailableSpaceUsages = Map.copyOf(mostAvailableUsagesBuilder); + maxHeapPerNode = Map.copyOf(maxHeapPerNodeBuilder); } @Override @@ -341,6 +345,7 @@ public void onFailure(Exception e) { } leastAvailableSpaceUsages = Map.of(); mostAvailableSpaceUsages = Map.of(); + maxHeapPerNode = Map.of(); } }, fetchRefs.acquire())); } @@ -433,6 +438,13 @@ private boolean shouldRefresh() { @Override public ClusterInfo getClusterInfo() { final IndicesStatsSummary indicesStatsSummary = this.indicesStatsSummary; // single volatile read + final Map shardHeapUsages = new HashMap<>(); + maxHeapPerNode.forEach((nodeId, maxHeapSize) -> { + final Long estimatedHeapUsage = estimatedHeapUsagePerNode.get(nodeId); + if (estimatedHeapUsage != null) { + shardHeapUsages.put(nodeId, new ShardHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage)); + } + }); return new ClusterInfo( leastAvailableSpaceUsages, mostAvailableSpaceUsages, @@ -503,10 +515,11 @@ static void buildShardLevelInfo( } } - private static void fillDiskUsagePerNode( + private static void processNodeStatsArray( List nodeStatsArray, Map newLeastAvailableUsages, - Map newMostAvailableUsages + Map newMostAvailableUsages, + Map maxHeapPerNodeBuilder ) { for (NodeStats nodeStats : nodeStatsArray) { DiskUsage leastAvailableUsage = DiskUsage.findLeastAvailablePath(nodeStats); @@ -517,6 +530,7 @@ private static void fillDiskUsagePerNode( if (mostAvailableUsage != null) { newMostAvailableUsages.put(nodeStats.getNode().getId(), mostAvailableUsage); } + maxHeapPerNodeBuilder.put(nodeStats.getNode().getId(), nodeStats.getJvm().getMem().getHeapMax()); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java index fb2b951ec92b2..3da97ac946f57 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java @@ -18,7 +18,7 @@ /** * Record representing an estimate of the heap used by allocated shards and ongoing merges on a particular node */ -public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, long estimatedUsageBytes) implements Writeable { +public record ShardHeapUsage(String nodeId, long totalBytes, long estimatedUsageBytes) implements Writeable { public ShardHeapUsage { assert totalBytes >= 0; @@ -26,13 +26,12 @@ public record ShardHeapUsage(String nodeId, String nodeName, long totalBytes, lo } public ShardHeapUsage(StreamInput in) throws IOException { - this(in.readString(), in.readString(), in.readVLong(), in.readVLong()); + this(in.readString(), in.readVLong(), in.readVLong()); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(this.nodeId); - out.writeString(this.nodeName); out.writeVLong(this.totalBytes); out.writeVLong(this.estimatedUsageBytes); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java index 2218300d13f1d..c3f3213e035ad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java @@ -13,17 +13,24 @@ import java.util.Map; +/** + * Collect the shard heap usage for each node in the cluster. + *

+ * Results are returned as a map of node ID to estimated heap usage in bytes + * + * @see ShardHeapUsage + */ public interface ShardHeapUsageCollector { /** - * This will be used when there are no heap usage suppliers available + * This will be used when there is no ShardHeapUsageCollector available */ ShardHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of()); /** - * Collect the heap usage for every node in the cluster + * Collect the shard heap usage for every node in the cluster * * @param listener The listener which will receive the results */ - void collectClusterHeapUsage(ActionListener> listener); + void collectClusterHeapUsage(ActionListener> listener); } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java index 5de885dabc674..051a44b7730e5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java @@ -53,7 +53,6 @@ private static Map randomNodeHeapUsage() { String key = randomAlphaOfLength(32); final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE); final ShardHeapUsage shardHeapUsage = new ShardHeapUsage( - randomAlphaOfLength(4), randomAlphaOfLength(4), totalBytes, randomIntBetween(0, totalBytes) diff --git a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java index 17721c2454e87..67a745e743b04 100644 --- a/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -145,7 +145,7 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { private static class StubShardShardHeapUsageCollector implements ShardHeapUsageCollector { @Override - public void collectClusterHeapUsage(ActionListener> listener) { + public void collectClusterHeapUsage(ActionListener> listener) { listener.onResponse(Map.of()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java index fd557b00e0503..f41cc8fafd887 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ShardHeapUsageTests.java @@ -19,7 +19,7 @@ public class ShardHeapUsageTests extends ESTestCase { public void testEstimatedUsageAsPercentage() { final long totalBytes = randomNonNegativeLong(); final long estimatedUsageBytes = randomLongBetween(0, totalBytes); - final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), randomIdentifier(), totalBytes, estimatedUsageBytes); + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), totalBytes, estimatedUsageBytes); assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), greaterThanOrEqualTo(0.0)); assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), lessThanOrEqualTo(100.0)); assertEquals(shardHeapUsage.estimatedUsageAsPercentage(), 100.0 * estimatedUsageBytes / totalBytes, 0.0001); @@ -29,7 +29,7 @@ public void testEstimatedFreeBytesAsPercentage() { final long totalBytes = randomNonNegativeLong(); final long estimatedUsageBytes = randomLongBetween(0, totalBytes); final long estimatedFreeBytes = totalBytes - estimatedUsageBytes; - final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), randomIdentifier(), totalBytes, estimatedUsageBytes); + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomUUID(), totalBytes, estimatedUsageBytes); assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), greaterThanOrEqualTo(0.0)); assertThat(shardHeapUsage.estimatedFreeBytesAsPercentage(), lessThanOrEqualTo(100.0)); assertEquals(shardHeapUsage.estimatedFreeBytesAsPercentage(), 100.0 * estimatedFreeBytes / totalBytes, 0.0001); From b8387bb33b5f9de3e01adc3836173584a6bfa0fb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 11 Jun 2025 02:35:05 +0000 Subject: [PATCH 29/30] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/cluster/ClusterInfoTests.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java index 051a44b7730e5..532f9b649fbf4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java @@ -52,11 +52,7 @@ private static Map randomNodeHeapUsage() { for (int i = 0; i < numEntries; i++) { String key = randomAlphaOfLength(32); final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE); - final ShardHeapUsage shardHeapUsage = new ShardHeapUsage( - randomAlphaOfLength(4), - totalBytes, - randomIntBetween(0, totalBytes) - ); + final ShardHeapUsage shardHeapUsage = new ShardHeapUsage(randomAlphaOfLength(4), totalBytes, randomIntBetween(0, totalBytes)); nodeHeapUsage.put(key, shardHeapUsage); } return nodeHeapUsage; From 26dba4d320a9b39ca7f9e262490dcf28929918d6 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 11 Jun 2025 12:47:06 +1000 Subject: [PATCH 30/30] Fix build --- .../cluster/InternalClusterInfoService.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 43c3730196b91..4c8655118dd82 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -90,7 +90,7 @@ public class InternalClusterInfoService implements ClusterInfoService, ClusterSt private volatile Map leastAvailableSpaceUsages; private volatile Map mostAvailableSpaceUsages; private volatile Map maxHeapPerNode; - private volatile Map estimatedHeapUsagePerNode; + private volatile Map shardHeapUsagePerNode; private volatile IndicesStatsSummary indicesStatsSummary; private final ThreadPool threadPool; @@ -114,7 +114,8 @@ public InternalClusterInfoService( ) { this.leastAvailableSpaceUsages = Map.of(); this.mostAvailableSpaceUsages = Map.of(); - this.shardHeapUsages = Map.of(); + this.maxHeapPerNode = Map.of(); + this.shardHeapUsagePerNode = Map.of(); this.indicesStatsSummary = IndicesStatsSummary.EMPTY; this.threadPool = threadPool; this.client = client; @@ -205,13 +206,13 @@ private void fetchNodesHeapUsage() { shardHeapUsageCollector.collectClusterHeapUsage(ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(Map currentShardHeapUsages) { - estimatedHeapUsagePerNode = currentShardHeapUsages; + shardHeapUsagePerNode = currentShardHeapUsages; } @Override public void onFailure(Exception e) { logger.warn("failed to fetch heap usage for nodes", e); - estimatedHeapUsagePerNode = Map.of(); + shardHeapUsagePerNode = Map.of(); } }, fetchRefs.acquire())); } @@ -440,7 +441,7 @@ public ClusterInfo getClusterInfo() { final IndicesStatsSummary indicesStatsSummary = this.indicesStatsSummary; // single volatile read final Map shardHeapUsages = new HashMap<>(); maxHeapPerNode.forEach((nodeId, maxHeapSize) -> { - final Long estimatedHeapUsage = estimatedHeapUsagePerNode.get(nodeId); + final Long estimatedHeapUsage = shardHeapUsagePerNode.get(nodeId); if (estimatedHeapUsage != null) { shardHeapUsages.put(nodeId, new ShardHeapUsage(nodeId, maxHeapSize.getBytes(), estimatedHeapUsage)); }