From 68fc2bf907a111e1aa90c9fe14e8ee70f02bb132 Mon Sep 17 00:00:00 2001 From: David Zane Date: Tue, 14 Apr 2026 20:00:22 -0700 Subject: [PATCH 1/5] Use binary serde for resource usage headers Signed-off-by: David Zane --- .../tasks/TaskResourceTrackingService.java | 64 ++++++++++---- .../TaskResourceTrackingServiceTests.java | 87 +++++++++++++++++-- 2 files changed, 125 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index 645e8e93bda04..ccfeacc4a260a 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -17,14 +17,15 @@ import org.opensearch.action.search.SearchShardTask; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ConcurrentMapLong; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.tasks.resourcetracker.ResourceStats; import org.opensearch.core.tasks.resourcetracker.ResourceStatsType; import org.opensearch.core.tasks.resourcetracker.ResourceUsageInfo; @@ -32,16 +33,13 @@ import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; import org.opensearch.core.tasks.resourcetracker.ThreadResourceInfo; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; @@ -322,7 +320,7 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { ) .build(); // Remove the existing TASK_RESOURCE_USAGE header since it would have come from an earlier phase in the same request. - threadPool.getThreadContext().updateResponseHeader(TASK_RESOURCE_USAGE, taskResourceInfo.toString()); + threadPool.getThreadContext().updateResponseHeader(TASK_RESOURCE_USAGE, serializeToBase64(taskResourceInfo)); } catch (Exception e) { logger.debug("Error during writing task resource usage: ", e); } @@ -336,31 +334,59 @@ public void removeTaskResourceUsage() { } /** - * Get the task resource usages from {@link ThreadContext} + * Get the task resource usages from {@link ThreadContext}. + *

+ * Deserializes from Base64-encoded binary format. If deserialization fails (e.g., due to a legacy + * JSON-formatted header from an older node during a rolling upgrade), the failure is handled gracefully + * and null is returned. The resource usage data for that shard is simply skipped. * - * @return {@link TaskResourceInfo} + * @return {@link TaskResourceInfo} or null if the header is absent or cannot be deserialized */ public TaskResourceInfo getTaskResourceUsageFromThreadContext() { List taskResourceUsages = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE); if (taskResourceUsages != null && taskResourceUsages.size() > 0) { String usage = taskResourceUsages.get(0); try { - if (usage != null && !usage.isEmpty()) { - XContentParser parser = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - new BytesArray(usage), - MediaTypeRegistry.JSON - ); - return TaskResourceInfo.PARSER.apply(parser, null); + if (usage != null && usage.isEmpty() == false) { + return deserializeFromBase64(usage); } - } catch (IOException e) { - logger.debug("fail to parse phase resource usages: ", e); + } catch (Exception e) { + logger.debug("failed to parse task resource usage header, skipping: ", e); } } return null; } + /** + * Serializes a {@link TaskResourceInfo} to a Base64-encoded binary string for use in response headers. + * Uses the efficient binary serialization instead of JSON to reduce CPU overhead. + * + * @param taskResourceInfo the task resource info to serialize + * @return a Base64-encoded string + * @throws IOException if serialization fails + */ + static String serializeToBase64(TaskResourceInfo taskResourceInfo) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + taskResourceInfo.writeTo(out); + byte[] bytes = BytesReference.toBytes(out.bytes()); + return Base64.getEncoder().encodeToString(bytes); + } + } + + /** + * Deserializes a {@link TaskResourceInfo} from a Base64-encoded binary string. + * + * @param headerValue the Base64-encoded header value + * @return the deserialized {@link TaskResourceInfo} + * @throws IOException if deserialization fails + */ + static TaskResourceInfo deserializeFromBase64(String headerValue) throws IOException { + byte[] bytes = Base64.getDecoder().decode(headerValue); + try (StreamInput in = StreamInput.wrap(bytes)) { + return TaskResourceInfo.readFromStream(in); + } + } + /** * Listener that gets invoked when a task execution completes. */ diff --git a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java index 823b52f1080ec..2c3fb91556f94 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java @@ -18,6 +18,7 @@ import org.opensearch.core.tasks.resourcetracker.ResourceStatsType; import org.opensearch.core.tasks.resourcetracker.ResourceUsageMetric; import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; +import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; import org.opensearch.core.tasks.resourcetracker.ThreadResourceInfo; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -164,17 +165,12 @@ public void testWriteTaskResourceUsage() { } public void testGetTaskResourceUsageFromThreadContext() { + // Legacy JSON format is no longer supported — should return null gracefully String taskResourceUsageJson = "{\"action\":\"testAction\",\"taskId\":1,\"parentTaskId\":2,\"nodeId\":\"nodeId\",\"taskResourceUsage\":{\"cpu_time_in_nanos\":1000,\"memory_in_bytes\":2000}}"; threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, taskResourceUsageJson); TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); - assertNotNull(result); - assertEquals("testAction", result.getAction()); - assertEquals(1L, result.getTaskId()); - assertEquals(2L, result.getParentTaskId()); - assertEquals("nodeId", result.getNodeId()); - assertEquals(1000L, result.getTaskResourceUsage().getCpuTimeInNanos()); - assertEquals(2000L, result.getTaskResourceUsage().getMemoryInBytes()); + assertNull("Legacy JSON header should return null since binary format is now required", result); } private void verifyThreadContextFixedHeaders(String key, String value) { @@ -183,4 +179,81 @@ private void verifyThreadContextFixedHeaders(String key, String value) { assertEquals(threadPool.getThreadContext().getResponseHeaders().get(key).get(0), value); } + public void testSerializeToBase64RoundTrip() throws Exception { + TaskResourceInfo original = new TaskResourceInfo( + "indices:data/read/search[phase/query]", + 12345L, + 67890L, + "U0rMsZg9RGOxdnVfMtMNVA", + new TaskResourceUsage(15000000L, 2048000L) + ); + + String encoded = TaskResourceTrackingService.serializeToBase64(original); + TaskResourceInfo deserialized = TaskResourceTrackingService.deserializeFromBase64(encoded); + assertEquals(original, deserialized); + } + + public void testBinaryHeaderReadFromThreadContext() throws Exception { + TaskResourceInfo original = new TaskResourceInfo( + "indices:data/read/search[phase/query]", + 1L, + 2L, + "nodeA", + new TaskResourceUsage(1000L, 2000L) + ); + + String binaryHeader = TaskResourceTrackingService.serializeToBase64(original); + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binaryHeader); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals(original, result); + } + + public void testJsonFallbackReturnsNullGracefully() { + // Simulate a header from an older node that uses JSON format — should fail gracefully + String jsonHeader = + "{\"action\":\"testAction\",\"taskId\":1,\"parentTaskId\":2,\"nodeId\":\"nodeId\",\"taskResourceUsage\":{\"cpu_time_in_nanos\":1000,\"memory_in_bytes\":2000}}"; + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, jsonHeader); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNull("Legacy JSON header should return null gracefully without binary prefix", result); + } + + public void testCorruptedHeaderReturnsNullGracefully() { + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, "not-valid-base64!!!"); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNull("Corrupted header should return null gracefully", result); + } + + public void testEmptyHeaderReturnsNull() { + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, ""); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNull(result); + } + + public void testWriteTaskResourceUsageUsesBinaryFormat() { + SearchShardTask task = new SearchShardTask(1, "test", "test", "task", TaskId.EMPTY_TASK_ID, new HashMap<>()); + taskResourceTrackingService.setTaskResourceTrackingEnabled(true); + taskResourceTrackingService.startTracking(task); + task.startThreadResourceTracking( + Thread.currentThread().threadId(), + ResourceStatsType.WORKER_STATS, + new ResourceUsageMetric(CPU, 100), + new ResourceUsageMetric(MEMORY, 100) + ); + taskResourceTrackingService.writeTaskResourceUsage(task, "node_1"); + Map> headers = threadPool.getThreadContext().getResponseHeaders(); + String headerValue = headers.get(TASK_RESOURCE_USAGE).get(0); + // Binary Base64 headers should not start with '{' (JSON) + assertFalse("Header should not be JSON format", headerValue.startsWith("{")); + + // Verify it can be deserialized back + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals("node_1", result.getNodeId()); + } + } From ffe1e5ad41cd19032963eb3248243e736d0d2197 Mon Sep 17 00:00:00 2001 From: David Zane Date: Wed, 20 May 2026 14:16:11 -0700 Subject: [PATCH 2/5] Add json option for bwc Signed-off-by: David Zane --- .../main/java/org/opensearch/node/Node.java | 3 +- .../tasks/TaskResourceTrackingService.java | 91 ++++++-- .../TaskResourceTrackingServiceTests.java | 208 ++++++++++++++++-- 3 files changed, 265 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 335e49d51a814..2c0b180a117af 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1059,7 +1059,8 @@ protected Node(final Environment initialEnvironment, Collection clas final TaskResourceTrackingService taskResourceTrackingService = new TaskResourceTrackingService( settings, clusterService.getClusterSettings(), - threadPool + threadPool, + clusterService ); final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index ccfeacc4a260a..f1fa6a5fcbe01 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -14,7 +14,10 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; +import org.opensearch.Version; import org.opensearch.action.search.SearchShardTask; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -24,6 +27,8 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ConcurrentMapLong; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.tasks.resourcetracker.ResourceStats; @@ -33,6 +38,10 @@ import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; import org.opensearch.core.tasks.resourcetracker.ThreadResourceInfo; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.RunnableTaskExecutionListener; import org.opensearch.threadpool.ThreadPool; @@ -63,20 +72,37 @@ public class TaskResourceTrackingService implements RunnableTaskExecutionListene public static final String TASK_ID = "TASK_ID"; public static final String TASK_RESOURCE_USAGE = "TASK_RESOURCE_USAGE"; + // First OpenSearch version that emits and parses the Base64 binary form of the + // TASK_RESOURCE_USAGE response header. Coordinators older than this only understand JSON. + static final Version BINARY_RESOURCE_USAGE_HEADER_VERSION = Version.V_3_7_0; + private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); private final ConcurrentMapLong resourceAwareTasks = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency(); private final List taskCompletionListeners = new ArrayList<>(); private final ThreadPool threadPool; + private final ClusterService clusterService; private volatile boolean taskResourceTrackingEnabled; @Inject - public TaskResourceTrackingService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) { + public TaskResourceTrackingService( + Settings settings, + ClusterSettings clusterSettings, + ThreadPool threadPool, + ClusterService clusterService + ) { this.taskResourceTrackingEnabled = TASK_RESOURCE_TRACKING_ENABLED.get(settings); this.threadPool = threadPool; + this.clusterService = clusterService; clusterSettings.addSettingsUpdateConsumer(TASK_RESOURCE_TRACKING_ENABLED, this::setTaskResourceTrackingEnabled); } + // Without ClusterService, writeTaskResourceUsage cannot verify the coordinator's version + // and will always emit the legacy JSON header. Forgoes the binary CPU win but is rolling-upgrade safe. + public TaskResourceTrackingService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) { + this(settings, clusterSettings, threadPool, null); + } + public void setTaskResourceTrackingEnabled(boolean taskResourceTrackingEnabled) { this.taskResourceTrackingEnabled = taskResourceTrackingEnabled; } @@ -319,13 +345,33 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { ) ) .build(); + String headerValue = canCoordinatorReadBinaryHeader(task) ? serializeToBase64(taskResourceInfo) : taskResourceInfo.toString(); // Remove the existing TASK_RESOURCE_USAGE header since it would have come from an earlier phase in the same request. - threadPool.getThreadContext().updateResponseHeader(TASK_RESOURCE_USAGE, serializeToBase64(taskResourceInfo)); + threadPool.getThreadContext().updateResponseHeader(TASK_RESOURCE_USAGE, headerValue); } catch (Exception e) { logger.debug("Error during writing task resource usage: ", e); } } + // Returns true only with positive confirmation that the coordinator can parse the binary header. + // Any unknown — no ClusterService, empty parent task, coordinator missing from cluster state — falls + // back to JSON, since the cost of an unwarranted JSON emission is small and bounded, while the cost + // of an unwarranted binary emission to an old coordinator is a hung search. + private boolean canCoordinatorReadBinaryHeader(SearchShardTask task) { + if (clusterService == null) { + return false; + } + String coordinatorNodeId = task.getParentTaskId().getNodeId(); + if (coordinatorNodeId.isEmpty()) { + return false; + } + DiscoveryNode coordinator = clusterService.state().nodes().get(coordinatorNodeId); + if (coordinator == null) { + return false; + } + return coordinator.getVersion().onOrAfter(BINARY_RESOURCE_USAGE_HEADER_VERSION); + } + /** * Remove the current task level resource usages. */ @@ -336,25 +382,30 @@ public void removeTaskResourceUsage() { /** * Get the task resource usages from {@link ThreadContext}. *

- * Deserializes from Base64-encoded binary format. If deserialization fails (e.g., due to a legacy - * JSON-formatted header from an older node during a rolling upgrade), the failure is handled gracefully - * and null is returned. The resource usage data for that shard is simply skipped. + * Tries the new Base64 binary format first; on failure falls back to the legacy JSON form so + * that resource usage from older data nodes is preserved during a rolling upgrade. * * @return {@link TaskResourceInfo} or null if the header is absent or cannot be deserialized */ public TaskResourceInfo getTaskResourceUsageFromThreadContext() { List taskResourceUsages = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE); - if (taskResourceUsages != null && taskResourceUsages.size() > 0) { - String usage = taskResourceUsages.get(0); + if (taskResourceUsages == null || taskResourceUsages.isEmpty()) { + return null; + } + String usage = taskResourceUsages.get(0); + if (usage == null || usage.isEmpty()) { + return null; + } + try { + return deserializeFromBase64(usage); + } catch (Exception binaryFailure) { try { - if (usage != null && usage.isEmpty() == false) { - return deserializeFromBase64(usage); - } - } catch (Exception e) { - logger.debug("failed to parse task resource usage header, skipping: ", e); + return deserializeFromJson(usage); + } catch (Exception jsonFailure) { + logger.debug("failed to parse task resource usage header (binary and JSON), skipping: ", jsonFailure); + return null; } } - return null; } /** @@ -387,6 +438,20 @@ static TaskResourceInfo deserializeFromBase64(String headerValue) throws IOExcep } } + // Legacy JSON parser for headers emitted by pre-V_3_7_0 data nodes during rolling upgrade. + static TaskResourceInfo deserializeFromJson(String headerValue) throws IOException { + try ( + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(headerValue), + MediaTypeRegistry.JSON + ) + ) { + return TaskResourceInfo.PARSER.apply(parser, null); + } + } + /** * Listener that gets invoked when a task execution completes. */ diff --git a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java index 2c3fb91556f94..bcd2374f84a83 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java @@ -8,9 +8,15 @@ package org.opensearch.tasks; +import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.tasks.TransportTasksActionTests; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchTask; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; @@ -20,12 +26,14 @@ import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; import org.opensearch.core.tasks.resourcetracker.ThreadResourceInfo; +import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,24 +48,71 @@ public class TaskResourceTrackingServiceTests extends OpenSearchTestCase { + private static final String LOCAL_NODE_ID = "local_node"; + private static final String OLD_COORDINATOR_ID = "old_coord_node"; + private static final String NEW_COORDINATOR_ID = "new_coord_node"; + private ThreadPool threadPool; + private ClusterService clusterService; private TaskResourceTrackingService taskResourceTrackingService; @Before public void setup() { threadPool = new TestThreadPool(TransportTasksActionTests.class.getSimpleName(), new AtomicReference<>()); + DiscoveryNode localNode = new DiscoveryNode( + LOCAL_NODE_ID, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + clusterService = ClusterServiceUtils.createClusterService(threadPool, localNode); taskResourceTrackingService = new TaskResourceTrackingService( Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - threadPool + threadPool, + clusterService ); } @After public void terminateThreadPool() { + clusterService.close(); terminate(threadPool); } + private void addCoordinatorNodeWithVersion(String nodeId, Version version) { + DiscoveryNode coordinator = new DiscoveryNode( + nodeId, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + version + ); + ClusterState current = clusterService.state(); + DiscoveryNodes updated = DiscoveryNodes.builder(current.nodes()).add(coordinator).build(); + ClusterState newState = ClusterState.builder(current).nodes(updated).build(); + ClusterServiceUtils.setState(clusterService, newState); + } + + private SearchShardTask searchShardTaskWithParent(String coordinatorNodeId) { + TaskId parent = new TaskId(coordinatorNodeId, randomLong()); + SearchShardTask task = new SearchShardTask(randomLong(), "test", "test", "task", parent, new HashMap<>()); + taskResourceTrackingService.setTaskResourceTrackingEnabled(true); + taskResourceTrackingService.startTracking(task); + task.startThreadResourceTracking( + Thread.currentThread().threadId(), + ResourceStatsType.WORKER_STATS, + new ResourceUsageMetric(CPU, 100), + new ResourceUsageMetric(MEMORY, 100) + ); + return task; + } + + private static String legacyJsonOf(TaskResourceInfo info) { + return info.toString(); + } + public void testThreadContextUpdateOnTrackingStart() { taskResourceTrackingService.setTaskResourceTrackingEnabled(true); @@ -164,13 +219,14 @@ public void testWriteTaskResourceUsage() { assertTrue(headers.containsKey(TASK_RESOURCE_USAGE)); } - public void testGetTaskResourceUsageFromThreadContext() { - // Legacy JSON format is no longer supported — should return null gracefully - String taskResourceUsageJson = - "{\"action\":\"testAction\",\"taskId\":1,\"parentTaskId\":2,\"nodeId\":\"nodeId\",\"taskResourceUsage\":{\"cpu_time_in_nanos\":1000,\"memory_in_bytes\":2000}}"; - threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, taskResourceUsageJson); + public void testGetTaskResourceUsageFromThreadContext() throws Exception { + TaskResourceInfo expected = new TaskResourceInfo("testAction", 1L, 2L, "nodeId", new TaskResourceUsage(1000L, 2000L)); + String binary = TaskResourceTrackingService.serializeToBase64(expected); + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); - assertNull("Legacy JSON header should return null since binary format is now required", result); + assertNotNull(result); + assertEquals(expected, result); } private void verifyThreadContextFixedHeaders(String key, String value) { @@ -210,16 +266,6 @@ public void testBinaryHeaderReadFromThreadContext() throws Exception { assertEquals(original, result); } - public void testJsonFallbackReturnsNullGracefully() { - // Simulate a header from an older node that uses JSON format — should fail gracefully - String jsonHeader = - "{\"action\":\"testAction\",\"taskId\":1,\"parentTaskId\":2,\"nodeId\":\"nodeId\",\"taskResourceUsage\":{\"cpu_time_in_nanos\":1000,\"memory_in_bytes\":2000}}"; - threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, jsonHeader); - - TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); - assertNull("Legacy JSON header should return null gracefully without binary prefix", result); - } - public void testCorruptedHeaderReturnsNullGracefully() { threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, "not-valid-base64!!!"); @@ -234,7 +280,9 @@ public void testEmptyHeaderReturnsNull() { assertNull(result); } - public void testWriteTaskResourceUsageUsesBinaryFormat() { + public void testWriteTaskResourceUsageRoundTrips() { + // EMPTY_TASK_ID means we can't resolve a coordinator, so the producer falls back to JSON; + // the consumer's JSON path must still round-trip cleanly. SearchShardTask task = new SearchShardTask(1, "test", "test", "task", TaskId.EMPTY_TASK_ID, new HashMap<>()); taskResourceTrackingService.setTaskResourceTrackingEnabled(true); taskResourceTrackingService.startTracking(task); @@ -245,15 +293,129 @@ public void testWriteTaskResourceUsageUsesBinaryFormat() { new ResourceUsageMetric(MEMORY, 100) ); taskResourceTrackingService.writeTaskResourceUsage(task, "node_1"); - Map> headers = threadPool.getThreadContext().getResponseHeaders(); - String headerValue = headers.get(TASK_RESOURCE_USAGE).get(0); - // Binary Base64 headers should not start with '{' (JSON) - assertFalse("Header should not be JSON format", headerValue.startsWith("{")); - // Verify it can be deserialized back TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); assertNotNull(result); assertEquals("node_1", result.getNodeId()); } + // Rolling-upgrade BC: one test per (data node, coordinator) quadrant. The TASK_RESOURCE_USAGE + // response header is one-way (data node -> coordinator), so the matrix is exhaustive. + // Quadrant 4 (old data -> old coord) is the pre-PR baseline; it's exercised by prior releases' + // tests and not reachable from this branch, so it has no test here. + + public void testBwc_NewDataNode_NewCoordinator() { + addCoordinatorNodeWithVersion(NEW_COORDINATOR_ID, Version.CURRENT); + + SearchShardTask task = searchShardTaskWithParent(NEW_COORDINATOR_ID); + taskResourceTrackingService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertFalse("expected binary header, got: " + headerValue, headerValue.startsWith("{")); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals(LOCAL_NODE_ID, result.getNodeId()); + } + + public void testBwc_NewDataNode_OldCoordinator() { + // If the data node emits binary here, the old coordinator's IOException-only catch lets + // XContentParseException escape, the shard response is silently dropped, and the search hangs. + addCoordinatorNodeWithVersion(OLD_COORDINATOR_ID, Version.CURRENT.minimumCompatibilityVersion()); + + SearchShardTask task = searchShardTaskWithParent(OLD_COORDINATOR_ID); + taskResourceTrackingService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertTrue("expected JSON header for old coordinator, got: " + headerValue, headerValue.startsWith("{")); + } + + public void testBwc_OldDataNode_NewCoordinator() { + TaskResourceInfo fromOldNode = new TaskResourceInfo( + "indices:data/read/search[phase/query]", + 42L, + 7L, + "old_data_node", + new TaskResourceUsage(1500L, 2500L) + ); + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, legacyJsonOf(fromOldNode)); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals(fromOldNode, result); + } + + public void testProducerEmitsJsonHeaderWhenCoordinatorMissingFromClusterState() { + // Coordinator's version is unknown (left the cluster, or state lag) — emit JSON for safety + // rather than risk a hang if the coordinator turns out to be old. + SearchShardTask task = searchShardTaskWithParent("ghost_coordinator"); + taskResourceTrackingService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertTrue("expected JSON when coordinator is unknown, got: " + headerValue, headerValue.startsWith("{")); + } + + public void testProducerEmitsJsonHeaderWhenNoParentTask() { + // No parent task means there's no coordinator to verify against — emit JSON for safety. + SearchShardTask task = new SearchShardTask(randomLong(), "test", "test", "task", TaskId.EMPTY_TASK_ID, new HashMap<>()); + taskResourceTrackingService.setTaskResourceTrackingEnabled(true); + taskResourceTrackingService.startTracking(task); + task.startThreadResourceTracking( + Thread.currentThread().threadId(), + ResourceStatsType.WORKER_STATS, + new ResourceUsageMetric(CPU, 100), + new ResourceUsageMetric(MEMORY, 100) + ); + taskResourceTrackingService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertTrue("expected JSON when no parent task, got: " + headerValue, headerValue.startsWith("{")); + } + + public void testProducerEmitsJsonHeaderWhenClusterServiceIsNull() { + // 3-arg constructor: no ClusterService, so the coordinator's version cannot be verified. + // Producer must default to JSON to remain rolling-upgrade safe. + TaskResourceTrackingService noClusterService = new TaskResourceTrackingService( + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + threadPool + ); + SearchShardTask task = new SearchShardTask( + randomLong(), + "test", + "test", + "task", + new TaskId(NEW_COORDINATOR_ID, randomLong()), + new HashMap<>() + ); + noClusterService.setTaskResourceTrackingEnabled(true); + noClusterService.startTracking(task); + task.startThreadResourceTracking( + Thread.currentThread().threadId(), + ResourceStatsType.WORKER_STATS, + new ResourceUsageMetric(CPU, 100), + new ResourceUsageMetric(MEMORY, 100) + ); + noClusterService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertTrue("expected JSON when ClusterService is null, got: " + headerValue, headerValue.startsWith("{")); + } + + public void testConsumerPrefersBinaryOverJsonFallback() throws Exception { + TaskResourceInfo expected = new TaskResourceInfo( + "indices:data/read/search[phase/query]", + 99L, + 33L, + "node_x", + new TaskResourceUsage(7777L, 8888L) + ); + String binary = TaskResourceTrackingService.serializeToBase64(expected); + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals(expected, result); + } + } From 3f95642537f942dae9572984bf5ae54dc9ed94d9 Mon Sep 17 00:00:00 2001 From: David Zane Date: Wed, 20 May 2026 14:45:19 -0700 Subject: [PATCH 3/5] Add dynamic enablement flag Signed-off-by: David Zane --- .../common/settings/ClusterSettings.java | 1 + .../tasks/TaskResourceTrackingService.java | 28 ++++++++++++++-- .../TaskResourceTrackingServiceTests.java | 32 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 66e011f82fe13..4c495844bc8ff 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -722,6 +722,7 @@ public void apply(Settings value, Settings current, Settings previous) { ShardIndexingPressureMemoryManager.MAX_OUTSTANDING_REQUESTS, IndexingPressure.MAX_INDEXING_BYTES, TaskResourceTrackingService.TASK_RESOURCE_TRACKING_ENABLED, + TaskResourceTrackingService.BINARY_RESOURCE_USAGE_HEADER_ENABLED, TaskManager.TASK_RESOURCE_CONSUMERS_ENABLED, TopNSearchTasksLogger.LOG_TOP_QUERIES_SIZE_SETTING, TopNSearchTasksLogger.LOG_TOP_QUERIES_FREQUENCY_SETTING, diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index f1fa6a5fcbe01..7bfc8ceaeb068 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -69,6 +69,14 @@ public class TaskResourceTrackingService implements RunnableTaskExecutionListene Setting.Property.Dynamic, Setting.Property.NodeScope ); + // Kill-switch for emitting the Base64 binary TASK_RESOURCE_USAGE response header. When false, + // data nodes always emit the legacy JSON header. The consumer always falls back to JSON regardless. + public static final Setting BINARY_RESOURCE_USAGE_HEADER_ENABLED = Setting.boolSetting( + "task_resource_tracking.binary_header.enabled", + true, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); public static final String TASK_ID = "TASK_ID"; public static final String TASK_RESOURCE_USAGE = "TASK_RESOURCE_USAGE"; @@ -83,6 +91,7 @@ public class TaskResourceTrackingService implements RunnableTaskExecutionListene private final ThreadPool threadPool; private final ClusterService clusterService; private volatile boolean taskResourceTrackingEnabled; + private volatile boolean binaryResourceUsageHeaderEnabled; @Inject public TaskResourceTrackingService( @@ -92,9 +101,11 @@ public TaskResourceTrackingService( ClusterService clusterService ) { this.taskResourceTrackingEnabled = TASK_RESOURCE_TRACKING_ENABLED.get(settings); + this.binaryResourceUsageHeaderEnabled = BINARY_RESOURCE_USAGE_HEADER_ENABLED.get(settings); this.threadPool = threadPool; this.clusterService = clusterService; clusterSettings.addSettingsUpdateConsumer(TASK_RESOURCE_TRACKING_ENABLED, this::setTaskResourceTrackingEnabled); + clusterSettings.addSettingsUpdateConsumer(BINARY_RESOURCE_USAGE_HEADER_ENABLED, this::setBinaryResourceUsageHeaderEnabled); } // Without ClusterService, writeTaskResourceUsage cannot verify the coordinator's version @@ -111,6 +122,14 @@ public boolean isTaskResourceTrackingEnabled() { return taskResourceTrackingEnabled; } + public void setBinaryResourceUsageHeaderEnabled(boolean binaryResourceUsageHeaderEnabled) { + this.binaryResourceUsageHeaderEnabled = binaryResourceUsageHeaderEnabled; + } + + public boolean isBinaryResourceUsageHeaderEnabled() { + return binaryResourceUsageHeaderEnabled; + } + public boolean isTaskResourceTrackingSupported() { return threadMXBean.isThreadAllocatedMemorySupported() && threadMXBean.isThreadAllocatedMemoryEnabled(); } @@ -354,10 +373,13 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { } // Returns true only with positive confirmation that the coordinator can parse the binary header. - // Any unknown — no ClusterService, empty parent task, coordinator missing from cluster state — falls - // back to JSON, since the cost of an unwarranted JSON emission is small and bounded, while the cost - // of an unwarranted binary emission to an old coordinator is a hung search. + // Any unknown — kill-switch off, no ClusterService, empty parent task, coordinator missing from + // cluster state — falls back to JSON, since the cost of an unwarranted JSON emission is small and + // bounded, while the cost of an unwarranted binary emission to an old coordinator is a hung search. private boolean canCoordinatorReadBinaryHeader(SearchShardTask task) { + if (binaryResourceUsageHeaderEnabled == false) { + return false; + } if (clusterService == null) { return false; } diff --git a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java index bcd2374f84a83..6f63ac343b1c4 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java @@ -402,6 +402,38 @@ public void testProducerEmitsJsonHeaderWhenClusterServiceIsNull() { assertTrue("expected JSON when ClusterService is null, got: " + headerValue, headerValue.startsWith("{")); } + public void testProducerEmitsJsonHeaderWhenBinaryHeaderKillSwitchIsOff() { + // Operator-controlled kill switch: when disabled, the producer must emit JSON even + // when the coordinator is on the current version and would otherwise accept binary. + addCoordinatorNodeWithVersion(NEW_COORDINATOR_ID, Version.CURRENT); + taskResourceTrackingService.setBinaryResourceUsageHeaderEnabled(false); + + SearchShardTask task = searchShardTaskWithParent(NEW_COORDINATOR_ID); + taskResourceTrackingService.writeTaskResourceUsage(task, LOCAL_NODE_ID); + + String headerValue = threadPool.getThreadContext().getResponseHeaders().get(TASK_RESOURCE_USAGE).get(0); + assertTrue("expected JSON when kill switch is off, got: " + headerValue, headerValue.startsWith("{")); + } + + public void testConsumerStillReadsBinaryWhenKillSwitchIsOff() throws Exception { + // The kill switch only gates the producer. The consumer must still decode binary so that + // a coordinator can read headers from data nodes that haven't yet picked up the flip. + taskResourceTrackingService.setBinaryResourceUsageHeaderEnabled(false); + TaskResourceInfo expected = new TaskResourceInfo( + "indices:data/read/search[phase/query]", + 5L, + 6L, + "data_node", + new TaskResourceUsage(11L, 22L) + ); + String binary = TaskResourceTrackingService.serializeToBase64(expected); + threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); + + TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); + assertNotNull(result); + assertEquals(expected, result); + } + public void testConsumerPrefersBinaryOverJsonFallback() throws Exception { TaskResourceInfo expected = new TaskResourceInfo( "indices:data/read/search[phase/query]", From f416371ede7416a9fc0a29ede92b1c8079e3b2e1 Mon Sep 17 00:00:00 2001 From: David Zane Date: Tue, 7 Jul 2026 13:27:24 -0700 Subject: [PATCH 4/5] Extract generic Writeable Base64 codec into shared helper Signed-off-by: David Zane --- .../common/io/stream/WriteableBase64.java | 65 ++++++++++++++ .../tasks/TaskResourceTrackingService.java | 41 ++------- .../io/stream/WriteableBase64Tests.java | 86 +++++++++++++++++++ .../TaskResourceTrackingServiceTests.java | 23 ++--- 4 files changed, 161 insertions(+), 54 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/io/stream/WriteableBase64.java create mode 100644 server/src/test/java/org/opensearch/common/io/stream/WriteableBase64Tests.java diff --git a/server/src/main/java/org/opensearch/common/io/stream/WriteableBase64.java b/server/src/main/java/org/opensearch/common/io/stream/WriteableBase64.java new file mode 100644 index 0000000000000..a23a848228859 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/io/stream/WriteableBase64.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.io.stream; + +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Base64; + +/** + * Utility for round-tripping a {@link Writeable} through its binary wire format encoded as a + * Base64 string. This is useful when a {@link Writeable} needs to travel over a channel that only + * carries strings (for example a {@code ThreadContext} response header), where the binary form is + * both cheaper to produce/consume than JSON and more compact once Base64-encoded. + * + * @opensearch.internal + */ +@InternalApi +public final class WriteableBase64 { + + private WriteableBase64() {} + + /** + * Serializes a {@link Writeable} to its binary wire format and Base64-encodes the result. + * + * @param writeable the object to serialize + * @return a Base64-encoded string of the object's binary representation + * @throws IOException if serialization fails + */ + public static String encode(Writeable writeable) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + writeable.writeTo(out); + byte[] bytes = BytesReference.toBytes(out.bytes()); + return Base64.getEncoder().encodeToString(bytes); + } + } + + /** + * Decodes a Base64 string produced by {@link #encode(Writeable)} back into an object using the + * supplied reader. + * + * @param encoded the Base64-encoded string + * @param reader reconstructs the object from a {@link StreamInput} (for example + * {@code MyType::readFromStream} or {@code MyType::new}) + * @param the type produced by {@code reader} + * @return the deserialized object + * @throws IOException if deserialization fails + * @throws IllegalArgumentException if the input is not valid Base64 + */ + public static T decode(String encoded, Writeable.Reader reader) throws IOException { + byte[] bytes = Base64.getDecoder().decode(encoded); + try (StreamInput in = StreamInput.wrap(bytes)) { + return reader.read(in); + } + } +} diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index 7bfc8ceaeb068..6782ac49ab6c5 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -20,7 +20,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.WriteableBase64; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -29,8 +29,6 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesArray; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.tasks.resourcetracker.ResourceStats; import org.opensearch.core.tasks.resourcetracker.ResourceStatsType; import org.opensearch.core.tasks.resourcetracker.ResourceUsageInfo; @@ -48,7 +46,6 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.ArrayList; -import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; @@ -364,7 +361,9 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { ) ) .build(); - String headerValue = canCoordinatorReadBinaryHeader(task) ? serializeToBase64(taskResourceInfo) : taskResourceInfo.toString(); + String headerValue = canCoordinatorReadBinaryHeader(task) + ? WriteableBase64.encode(taskResourceInfo) + : taskResourceInfo.toString(); // Remove the existing TASK_RESOURCE_USAGE header since it would have come from an earlier phase in the same request. threadPool.getThreadContext().updateResponseHeader(TASK_RESOURCE_USAGE, headerValue); } catch (Exception e) { @@ -419,7 +418,7 @@ public TaskResourceInfo getTaskResourceUsageFromThreadContext() { return null; } try { - return deserializeFromBase64(usage); + return WriteableBase64.decode(usage, TaskResourceInfo::readFromStream); } catch (Exception binaryFailure) { try { return deserializeFromJson(usage); @@ -430,36 +429,6 @@ public TaskResourceInfo getTaskResourceUsageFromThreadContext() { } } - /** - * Serializes a {@link TaskResourceInfo} to a Base64-encoded binary string for use in response headers. - * Uses the efficient binary serialization instead of JSON to reduce CPU overhead. - * - * @param taskResourceInfo the task resource info to serialize - * @return a Base64-encoded string - * @throws IOException if serialization fails - */ - static String serializeToBase64(TaskResourceInfo taskResourceInfo) throws IOException { - try (BytesStreamOutput out = new BytesStreamOutput()) { - taskResourceInfo.writeTo(out); - byte[] bytes = BytesReference.toBytes(out.bytes()); - return Base64.getEncoder().encodeToString(bytes); - } - } - - /** - * Deserializes a {@link TaskResourceInfo} from a Base64-encoded binary string. - * - * @param headerValue the Base64-encoded header value - * @return the deserialized {@link TaskResourceInfo} - * @throws IOException if deserialization fails - */ - static TaskResourceInfo deserializeFromBase64(String headerValue) throws IOException { - byte[] bytes = Base64.getDecoder().decode(headerValue); - try (StreamInput in = StreamInput.wrap(bytes)) { - return TaskResourceInfo.readFromStream(in); - } - } - // Legacy JSON parser for headers emitted by pre-V_3_7_0 data nodes during rolling upgrade. static TaskResourceInfo deserializeFromJson(String headerValue) throws IOException { try ( diff --git a/server/src/test/java/org/opensearch/common/io/stream/WriteableBase64Tests.java b/server/src/test/java/org/opensearch/common/io/stream/WriteableBase64Tests.java new file mode 100644 index 0000000000000..f14284be988c9 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/io/stream/WriteableBase64Tests.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.io.stream; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Base64; +import java.util.Objects; + +public class WriteableBase64Tests extends OpenSearchTestCase { + + /** A minimal {@link Writeable} used to exercise the codec independently of any production type. */ + private static class Sample implements Writeable { + private final String name; + private final long value; + + Sample(String name, long value) { + this.name = name; + this.value = value; + } + + Sample(StreamInput in) throws IOException { + this.name = in.readString(); + this.value = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeLong(value); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Sample sample = (Sample) o; + return value == sample.value && Objects.equals(name, sample.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } + } + + public void testRoundTrip() throws IOException { + Sample original = new Sample(randomAlphaOfLength(12), randomLong()); + + String encoded = WriteableBase64.encode(original); + Sample decoded = WriteableBase64.decode(encoded, Sample::new); + + assertEquals(original, decoded); + } + + public void testEncodeProducesValidBase64() throws IOException { + Sample original = new Sample("action", 42L); + + String encoded = WriteableBase64.encode(original); + + // Must decode as standard Base64 without throwing; the resulting bytes reconstruct the object. + byte[] decodedBytes = Base64.getDecoder().decode(encoded); + try (StreamInput in = StreamInput.wrap(decodedBytes)) { + assertEquals(original, new Sample(in)); + } + } + + public void testEmptyStringThrows() { + // An empty payload has no bytes for the reader to consume. + expectThrows(Exception.class, () -> WriteableBase64.decode("", Sample::new)); + } + + public void testInvalidBase64Throws() { + expectThrows(IllegalArgumentException.class, () -> WriteableBase64.decode("not-valid-base64!!!", Sample::new)); + } +} diff --git a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java index 6f63ac343b1c4..035cbfcc46924 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResourceTrackingServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.io.stream.WriteableBase64; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; @@ -221,7 +222,7 @@ public void testWriteTaskResourceUsage() { public void testGetTaskResourceUsageFromThreadContext() throws Exception { TaskResourceInfo expected = new TaskResourceInfo("testAction", 1L, 2L, "nodeId", new TaskResourceUsage(1000L, 2000L)); - String binary = TaskResourceTrackingService.serializeToBase64(expected); + String binary = WriteableBase64.encode(expected); threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); @@ -235,20 +236,6 @@ private void verifyThreadContextFixedHeaders(String key, String value) { assertEquals(threadPool.getThreadContext().getResponseHeaders().get(key).get(0), value); } - public void testSerializeToBase64RoundTrip() throws Exception { - TaskResourceInfo original = new TaskResourceInfo( - "indices:data/read/search[phase/query]", - 12345L, - 67890L, - "U0rMsZg9RGOxdnVfMtMNVA", - new TaskResourceUsage(15000000L, 2048000L) - ); - - String encoded = TaskResourceTrackingService.serializeToBase64(original); - TaskResourceInfo deserialized = TaskResourceTrackingService.deserializeFromBase64(encoded); - assertEquals(original, deserialized); - } - public void testBinaryHeaderReadFromThreadContext() throws Exception { TaskResourceInfo original = new TaskResourceInfo( "indices:data/read/search[phase/query]", @@ -258,7 +245,7 @@ public void testBinaryHeaderReadFromThreadContext() throws Exception { new TaskResourceUsage(1000L, 2000L) ); - String binaryHeader = TaskResourceTrackingService.serializeToBase64(original); + String binaryHeader = WriteableBase64.encode(original); threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binaryHeader); TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); @@ -426,7 +413,7 @@ public void testConsumerStillReadsBinaryWhenKillSwitchIsOff() throws Exception { "data_node", new TaskResourceUsage(11L, 22L) ); - String binary = TaskResourceTrackingService.serializeToBase64(expected); + String binary = WriteableBase64.encode(expected); threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); @@ -442,7 +429,7 @@ public void testConsumerPrefersBinaryOverJsonFallback() throws Exception { "node_x", new TaskResourceUsage(7777L, 8888L) ); - String binary = TaskResourceTrackingService.serializeToBase64(expected); + String binary = WriteableBase64.encode(expected); threadPool.getThreadContext().addResponseHeader(TASK_RESOURCE_USAGE, binary); TaskResourceInfo result = taskResourceTrackingService.getTaskResourceUsageFromThreadContext(); From 2e2537aef3386ad07924dc29d32be1c5cd52b378 Mon Sep 17 00:00:00 2001 From: David Zane Date: Wed, 8 Jul 2026 14:28:07 -0700 Subject: [PATCH 5/5] Gate binary resource usage header on 3.8.0 Signed-off-by: David Zane --- .../org/opensearch/tasks/TaskResourceTrackingService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index 6782ac49ab6c5..cd76d7b954362 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -79,7 +79,7 @@ public class TaskResourceTrackingService implements RunnableTaskExecutionListene // First OpenSearch version that emits and parses the Base64 binary form of the // TASK_RESOURCE_USAGE response header. Coordinators older than this only understand JSON. - static final Version BINARY_RESOURCE_USAGE_HEADER_VERSION = Version.V_3_7_0; + static final Version BINARY_RESOURCE_USAGE_HEADER_VERSION = Version.V_3_8_0; private static final ThreadMXBean threadMXBean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); @@ -376,7 +376,7 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { // cluster state — falls back to JSON, since the cost of an unwarranted JSON emission is small and // bounded, while the cost of an unwarranted binary emission to an old coordinator is a hung search. private boolean canCoordinatorReadBinaryHeader(SearchShardTask task) { - if (binaryResourceUsageHeaderEnabled == false) { + if (isBinaryResourceUsageHeaderEnabled() == false) { return false; } if (clusterService == null) { @@ -429,7 +429,7 @@ public TaskResourceInfo getTaskResourceUsageFromThreadContext() { } } - // Legacy JSON parser for headers emitted by pre-V_3_7_0 data nodes during rolling upgrade. + // Legacy JSON parser for headers emitted by pre-V_3_8_0 data nodes during rolling upgrade. static TaskResourceInfo deserializeFromJson(String headerValue) throws IOException { try ( XContentParser parser = XContentHelper.createParser(