From 28bc7f93a424dea50025ca02b280c0474c01fa22 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 11 Feb 2026 10:21:32 +0000 Subject: [PATCH 01/45] Initial plumbing - need to address TODOs --- .../BulkByScrollParallelizationHelper.java | 3 ++ .../org/elasticsearch/reindex/Reindexer.java | 34 ++++++++++++++++++- .../reindex/TransportReindexAction.java | 15 +++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java index 0b5be5e0f0b5b..704512ff2a608 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java @@ -91,6 +91,9 @@ static > void executeSliced DiscoveryNode node, Runnable workerAction ) { + // TODO: make this a utils method to decide because we MUST make PIT IFF we also use a PIT hit source (and vice versa) + // If feature flag: + // If valid PIT reasons -> open PIT if (task.isLeader()) { sendSubRequests(client, action, node.getId(), task, request, listener); } else if (task.isWorker()) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index fffd4e6e0f3c7..a831dfec92c87 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -113,9 +113,12 @@ public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListen BulkByScrollParallelizationHelper.initTaskState(task, request, client, listener); } - public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener listener) { + public void execute(BulkByScrollTask task, ReindexRequest request, int remoteVersion, Client bulkClient, ActionListener listener) { long startTime = System.nanoTime(); + // TODO - We need to decide inside executeSlicedAction whether we're using PIT or not and open the PIT if so + // Since the request (and request.getRemoteINfo()) and the remoteVersion can be passed in, then we just need an IF statement. + // TODO - Do we need to make this decision in a utils class somewhere since it will be mirrored when the hitsourceis created BulkByScrollParallelizationHelper.executeSlicedAction( task, request, @@ -136,6 +139,7 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl projectResolver.getProjectState(clusterService.state()), reindexSslConfig, request, + remoteVersion, wrapWithMetrics(listener, reindexMetrics, startTime, request.getRemoteInfo() != null) ); searchAction.start(); @@ -263,6 +267,8 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction createdThreads = emptyList(); + private int remoteVersion; + AsyncIndexBySearchAction( BulkByScrollTask task, Logger logger, @@ -273,6 +279,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener ) { super( @@ -294,6 +301,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction()); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index bc8acfd476c5b..3475d0e8cf8c7 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -40,6 +41,12 @@ public class TransportReindexAction extends HandledTransportAction listener) { validate(request); BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + + // TODO - Get remote version using async callback functions + int remoteVersion = 5; + reindexer.initTask( bulkByScrollTask, request, - listener.delegateFailure((l, v) -> reindexer.execute(bulkByScrollTask, request, getBulkClient(), l)) + listener.delegateFailure((l, v) -> { + reindexer.execute(bulkByScrollTask, request, remoteVersion, getBulkClient(), l); + }) ); } From 9427763c4911c75eb470ad95c0a43a878bbd2fb8 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 13 Feb 2026 11:58:12 +0000 Subject: [PATCH 02/45] Creates Reindexer.lookupRemoteVersion --- .../BulkByScrollParallelizationHelper.java | 3 - .../org/elasticsearch/reindex/Reindexer.java | 61 +++++-- .../reindex/TransportReindexAction.java | 22 ++- .../reindex/remote/RemoteReindexingUtils.java | 163 ++++++++++++++++++ 4 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java index 704512ff2a608..0b5be5e0f0b5b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java @@ -91,9 +91,6 @@ static > void executeSliced DiscoveryNode node, Runnable workerAction ) { - // TODO: make this a utils method to decide because we MUST make PIT IFF we also use a PIT hit source (and vice versa) - // If feature flag: - // If valid PIT reasons -> open PIT if (task.isLeader()) { sendSubRequests(client, action, node.getId(), task, request, listener); } else if (task.isWorker()) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index a831dfec92c87..b7465954644bb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -19,6 +19,7 @@ import org.apache.http.message.BasicHeader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -49,15 +50,18 @@ import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.ScrollableHitSource; import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState; +import org.elasticsearch.reindex.remote.RemoteReindexingUtils; import org.elasticsearch.reindex.remote.RemoteScrollableHitSource; import org.elasticsearch.script.CtxMap; import org.elasticsearch.script.ReindexMetadata; import org.elasticsearch.script.ReindexScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -90,6 +94,7 @@ public class Reindexer { private final ScriptService scriptService; private final ReindexSslConfig reindexSslConfig; private final ReindexMetrics reindexMetrics; + Version remoteVersion; Reindexer( ClusterService clusterService, @@ -109,16 +114,56 @@ public class Reindexer { this.reindexMetrics = reindexMetrics; } + /** + * If we're reindexing from a remote cluster, then we look up the remote version and save it to {@link #remoteVersion} + * Otherwise, we set {@link #remoteVersion} as null + * @param listener The listener to complete once we've determined the remote cluster version. + * This is typically the reindexing request itself. + */ + public void lookupRemoteVersion(Task task, ReindexRequest request, ActionListener listener) { + // If we're reindexing from a remote source, then we need to determine the remote version to decide whether we use + // scroll search or point-in-time search + if (request.getRemoteInfo() != null) { + RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { + @Override + public void onResponse(Version version) { + remoteVersion = version; + // The listener onResponse will call subsequent reindexing methods. However, the remoteVersion will now be present as a + // class variable rather than a parameter + listener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + public void onRejection(Exception e) { + // No rejection concept in ActionListener, therefore treating as failure + listener.onFailure(e); + } + }; + + RemoteInfo remoteInfo = request.getRemoteInfo(); + assert reindexSslConfig != null : "Reindex ssl config must be set"; + RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); + RemoteReindexingUtils.lookupRemoteVersion(rejectAwareListener, threadPool, restClient); + } + // We're reindexing from the same cluster, so we set the remote version to null + else { + remoteVersion = null; + listener.onResponse(null); + } + } + public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.initTaskState(task, request, client, listener); } - public void execute(BulkByScrollTask task, ReindexRequest request, int remoteVersion, Client bulkClient, ActionListener listener) { + public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener listener) { long startTime = System.nanoTime(); - // TODO - We need to decide inside executeSlicedAction whether we're using PIT or not and open the PIT if so - // Since the request (and request.getRemoteINfo()) and the remoteVersion can be passed in, then we just need an IF statement. - // TODO - Do we need to make this decision in a utils class somewhere since it will be mirrored when the hitsourceis created BulkByScrollParallelizationHelper.executeSlicedAction( task, request, @@ -139,7 +184,6 @@ public void execute(BulkByScrollTask task, ReindexRequest request, int remoteVer projectResolver.getProjectState(clusterService.state()), reindexSslConfig, request, - remoteVersion, wrapWithMetrics(listener, reindexMetrics, startTime, request.getRemoteInfo() != null) ); searchAction.start(); @@ -259,6 +303,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction createdThreads = emptyList(); - private int remoteVersion; - AsyncIndexBySearchAction( BulkByScrollTask task, Logger logger, @@ -279,7 +322,6 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener ) { super( @@ -301,7 +343,6 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction()); assert sslConfig != null : "Reindex ssl config must be set"; + // TODO - Remove? Remember, we only use this REST client for remote cases which is orthogonal to whether we use PIT RestClient restClient = buildRestClient(remoteInfo, sslConfig, task.getId(), createdThreads); return new RemoteScrollableHitSource( logger, diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index 3475d0e8cf8c7..2be8e4375acb3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -116,15 +116,21 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener { - reindexer.execute(bulkByScrollTask, request, remoteVersion, getBulkClient(), l); - }) + listener.delegateFailure( + (l, v) -> reindexer.initTask( + bulkByScrollTask, + request, + l.delegateFailure((l2, v2) -> reindexer.execute(bulkByScrollTask, request, getBulkClient(), l2)) + ) + ) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java new file mode 100644 index 0000000000000..3d59b99c5025e --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.reindex.remote; + +import org.apache.http.ContentTooLongException; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.reindex.RejectAwareActionListener; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.util.function.BiFunction; + +import static org.elasticsearch.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER; + +public class RemoteReindexingUtils { + + public static void lookupRemoteVersion(RejectAwareActionListener listener, ThreadPool threadPool, RestClient client) { + execute(new Request("GET", ""), MAIN_ACTION_PARSER, listener, threadPool, client); + } + + static void execute( + Request request, + BiFunction parser, + RejectAwareActionListener listener, + ThreadPool threadPool, + RestClient client + ) { + // Preserve the thread context so headers survive after the call + java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); + try { + client.performRequestAsync(request, new ResponseListener() { + @Override + public void onSuccess(org.elasticsearch.client.Response response) { + // Restore the thread context to get the precious headers + try (ThreadContext.StoredContext ctx = contextSupplier.get()) { + assert ctx != null; // eliminates compiler warning + T parsedResponse; + try { + HttpEntity responseEntity = response.getEntity(); + InputStream content = responseEntity.getContent(); + XContentType xContentType = null; + if (responseEntity.getContentType() != null) { + final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType(); + xContentType = XContentType.fromMediaType(mimeType); + } + if (xContentType == null) { + try { + throw new ElasticsearchException( + "Response didn't include Content-Type: " + bodyMessage(response.getEntity()) + ); + } catch (IOException e) { + ElasticsearchException ee = new ElasticsearchException("Error extracting body from response"); + ee.addSuppressed(e); + throw ee; + } + } + // EMPTY is safe here because we don't call namedObject + try ( + XContentParser xContentParser = xContentType.xContent() + .createParser( + XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE), + content + ) + ) { + parsedResponse = parser.apply(xContentParser, xContentType); + } catch (XContentParseException e) { + /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it + * is totally wrong and we're probably not talking to Elasticsearch. */ + throw new ElasticsearchException( + "Error parsing the response, remote is likely not an Elasticsearch instance", + e + ); + } + } catch (IOException e) { + throw new ElasticsearchException( + "Error deserializing response, remote is likely not an Elasticsearch instance", + e + ); + } + listener.onResponse(parsedResponse); + } + } + + @Override + public void onFailure(Exception e) { + try (ThreadContext.StoredContext ctx = contextSupplier.get()) { + assert ctx != null; // eliminates compiler warning + if (e instanceof ResponseException re) { + int statusCode = re.getResponse().getStatusLine().getStatusCode(); + e = wrapExceptionToPreserveStatus(statusCode, re.getResponse().getEntity(), re); + if (RestStatus.TOO_MANY_REQUESTS.getStatus() == statusCode) { + listener.onRejection(e); + return; + } + } else if (e instanceof ContentTooLongException) { + e = new IllegalArgumentException( + "Remote responded with a chunk that was too large. Use a smaller batch size.", + e + ); + } + listener.onFailure(e); + } + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Wrap the ResponseException in an exception that'll preserve its status code if possible so we can send it back to the user. We might + * not have a constant for the status code so in that case we just use 500 instead. We also extract make sure to include the response + * body in the message so the user can figure out *why* the remote Elasticsearch service threw the error back to us. + */ + static ElasticsearchStatusException wrapExceptionToPreserveStatus(int statusCode, @Nullable HttpEntity entity, Exception cause) { + RestStatus status = RestStatus.fromCode(statusCode); + String messagePrefix = ""; + if (status == null) { + messagePrefix = "Couldn't extract status [" + statusCode + "]. "; + status = RestStatus.INTERNAL_SERVER_ERROR; + } + try { + return new ElasticsearchStatusException(messagePrefix + bodyMessage(entity), status, cause); + } catch (IOException ioe) { + ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause); + e.addSuppressed(ioe); + return e; + } + } + + static String bodyMessage(@Nullable HttpEntity entity) throws IOException { + if (entity == null) { + return "No error body."; + } else { + return "body=" + EntityUtils.toString(entity); + } + } +} From 3430f18ee4f1194ed230012d7755d5da7254ccc8 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 13 Feb 2026 16:12:27 +0000 Subject: [PATCH 03/45] Refactor RemoteScrollableHitSource to use the utils class --- .../remote/RemoteScrollableHitSource.java | 151 +-------- .../remote/RemoteReindexingUtilsTests.java | 307 ++++++++++++++++++ .../RemoteScrollableHitSourceTests.java | 159 +++------ 3 files changed, 361 insertions(+), 256 deletions(-) create mode 100644 modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java index e2f899373a6cf..8ba2985d8b3ce 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java @@ -9,47 +9,31 @@ package org.elasticsearch.reindex.remote; -import org.apache.http.ContentTooLongException; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.Supplier; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.ResumeInfo.ScrollWorkerResumeInfo; import org.elasticsearch.index.reindex.ResumeInfo.WorkerResumeInfo; import org.elasticsearch.index.reindex.ScrollableHitSource; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xcontent.XContentParseException; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.io.InputStream; -import java.util.function.BiFunction; import java.util.function.Consumer; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.core.TimeValue.timeValueNanos; -import static org.elasticsearch.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER; +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.execute; +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.lookupRemoteVersion; import static org.elasticsearch.reindex.remote.RemoteResponseParsers.RESPONSE_PARSER; public class RemoteScrollableHitSource extends ScrollableHitSource { @@ -82,9 +66,11 @@ protected void doStart(RejectAwareActionListener searchListener) { execute( RemoteRequestBuilders.initialSearch(searchRequest, remote.getQuery(), remoteVersion), RESPONSE_PARSER, - RejectAwareActionListener.withResponseHandler(searchListener, r -> onStartResponse(searchListener, r)) + RejectAwareActionListener.withResponseHandler(searchListener, r -> onStartResponse(searchListener, r)), + threadPool, + client ); - })); + }), threadPool, client); } @Override @@ -96,11 +82,8 @@ public void restoreState(WorkerResumeInfo resumeInfo) { setScroll(scrollResumeInfo.scrollId()); } - void lookupRemoteVersion(RejectAwareActionListener listener) { - execute(new Request("GET", ""), MAIN_ACTION_PARSER, listener); - } - - private void onStartResponse(RejectAwareActionListener searchListener, Response response) { + // Exposed for testing + void onStartResponse(RejectAwareActionListener searchListener, Response response) { if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) { logger.debug("First response looks like a scan response. Jumping right to the second. scroll=[{}]", response.getScrollId()); doStartNextScroll(response.getScrollId(), timeValueMillis(0), searchListener); @@ -112,7 +95,7 @@ private void onStartResponse(RejectAwareActionListener searchListener, @Override protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, RejectAwareActionListener searchListener) { TimeValue keepAlive = timeValueNanos(searchRequest.scroll().nanos() + extraKeepAlive.nanos()); - execute(RemoteRequestBuilders.scroll(scrollId, keepAlive, remoteVersion), RESPONSE_PARSER, searchListener); + execute(RemoteRequestBuilders.scroll(scrollId, keepAlive, remoteVersion), RESPONSE_PARSER, searchListener, threadPool, client); } @Override @@ -166,120 +149,4 @@ protected void cleanup(Runnable onCompletion) { } }); } - - private void execute( - Request request, - BiFunction parser, - RejectAwareActionListener listener - ) { - // Preserve the thread context so headers survive after the call - java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); - try { - client.performRequestAsync(request, new ResponseListener() { - @Override - public void onSuccess(org.elasticsearch.client.Response response) { - // Restore the thread context to get the precious headers - try (ThreadContext.StoredContext ctx = contextSupplier.get()) { - assert ctx != null; // eliminates compiler warning - T parsedResponse; - try { - HttpEntity responseEntity = response.getEntity(); - InputStream content = responseEntity.getContent(); - XContentType xContentType = null; - if (responseEntity.getContentType() != null) { - final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType(); - xContentType = XContentType.fromMediaType(mimeType); - } - if (xContentType == null) { - try { - throw new ElasticsearchException( - "Response didn't include Content-Type: " + bodyMessage(response.getEntity()) - ); - } catch (IOException e) { - ElasticsearchException ee = new ElasticsearchException("Error extracting body from response"); - ee.addSuppressed(e); - throw ee; - } - } - // EMPTY is safe here because we don't call namedObject - try ( - XContentParser xContentParser = xContentType.xContent() - .createParser( - XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE), - content - ) - ) { - parsedResponse = parser.apply(xContentParser, xContentType); - } catch (XContentParseException e) { - /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it - * is totally wrong and we're probably not talking to Elasticsearch. */ - throw new ElasticsearchException( - "Error parsing the response, remote is likely not an Elasticsearch instance", - e - ); - } - } catch (IOException e) { - throw new ElasticsearchException( - "Error deserializing response, remote is likely not an Elasticsearch instance", - e - ); - } - listener.onResponse(parsedResponse); - } - } - - @Override - public void onFailure(Exception e) { - try (ThreadContext.StoredContext ctx = contextSupplier.get()) { - assert ctx != null; // eliminates compiler warning - if (e instanceof ResponseException re) { - int statusCode = re.getResponse().getStatusLine().getStatusCode(); - e = wrapExceptionToPreserveStatus(statusCode, re.getResponse().getEntity(), re); - if (RestStatus.TOO_MANY_REQUESTS.getStatus() == statusCode) { - listener.onRejection(e); - return; - } - } else if (e instanceof ContentTooLongException) { - e = new IllegalArgumentException( - "Remote responded with a chunk that was too large. Use a smaller batch size.", - e - ); - } - listener.onFailure(e); - } - } - }); - } catch (Exception e) { - listener.onFailure(e); - } - } - - /** - * Wrap the ResponseException in an exception that'll preserve its status code if possible so we can send it back to the user. We might - * not have a constant for the status code so in that case we just use 500 instead. We also extract make sure to include the response - * body in the message so the user can figure out *why* the remote Elasticsearch service threw the error back to us. - */ - static ElasticsearchStatusException wrapExceptionToPreserveStatus(int statusCode, @Nullable HttpEntity entity, Exception cause) { - RestStatus status = RestStatus.fromCode(statusCode); - String messagePrefix = ""; - if (status == null) { - messagePrefix = "Couldn't extract status [" + statusCode + "]. "; - status = RestStatus.INTERNAL_SERVER_ERROR; - } - try { - return new ElasticsearchStatusException(messagePrefix + bodyMessage(entity), status, cause); - } catch (IOException ioe) { - ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause); - e.addSuppressed(ioe); - return e; - } - } - - private static String bodyMessage(@Nullable HttpEntity entity) throws IOException { - if (entity == null) { - return "No error body."; - } else { - return "body=" + EntityUtils.toString(entity); - } - } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java new file mode 100644 index 0000000000000..5eaf4d4f1a8d8 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -0,0 +1,307 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.reindex.remote; + +import org.apache.http.HttpEntity; +import org.apache.http.RequestLine; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.reindex.RejectAwareActionListener; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.URL; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.wrapExceptionToPreserveStatus; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RemoteReindexingUtilsTests extends ESTestCase { + private ThreadPool threadPool; + private RestClient client; + + @Before + public void setUp() throws Exception { + super.setUp(); + threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return EsExecutors.DIRECT_EXECUTOR_SERVICE; + } + }; + client = mock(RestClient.class); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + terminate(threadPool); + } + + /** + * Verifies that lookupRemoteVersion correctly parses historical and + * forward-compatible main action responses. + */ + public void testLookupRemoteVersion() throws Exception { + assertLookupRemoteVersion(Version.fromString("0.20.5"), "main/0_20_5.json"); + assertLookupRemoteVersion(Version.fromString("0.90.13"), "main/0_90_13.json"); + assertLookupRemoteVersion(Version.fromString("1.7.5"), "main/1_7_5.json"); + assertLookupRemoteVersion(Version.fromId(2030399), "main/2_3_3.json"); + assertLookupRemoteVersion(Version.fromId(5000099), "main/5_0_0_alpha_3.json"); + assertLookupRemoteVersion(Version.fromId(5000099), "main/with_unknown_fields.json"); + } + + private void assertLookupRemoteVersion(Version expected, String resource) throws Exception { + AtomicBoolean called = new AtomicBoolean(); + URL url = Thread.currentThread().getContextClassLoader().getResource("responses/" + resource); + assertNotNull("missing test resource [" + resource + "]", url); + + HttpEntity entity = new InputStreamEntity(FileSystemUtils.openFileURLStream(url), ContentType.APPLICATION_JSON); + org.elasticsearch.client.Response response = mock(org.elasticsearch.client.Response.class); + when(response.getEntity()).thenReturn(entity); + + mockSuccess(response); + RemoteReindexingUtils.lookupRemoteVersion(RejectAwareActionListener.wrap(v -> { + assertEquals(expected, v); + called.set(true); + }, e -> fail(), e -> fail()), threadPool, client); + assertTrue("listener was not called", called.get()); + } + + /** + * Verifies that lookupRemoteVersion fails when the response does not include + * a Content-Type header, and that the error message includes the response body. + */ + public void testLookupRemoteVersionFailsWithoutContentType() throws Exception { + URL url = Thread.currentThread().getContextClassLoader().getResource("responses/main/0_20_5.json"); + assertNotNull(url); + + HttpEntity entity = new InputStreamEntity( + FileSystemUtils.openFileURLStream(url), + // intentionally no Content-Type + null + ); + + org.elasticsearch.client.Response response = mock(org.elasticsearch.client.Response.class); + when(response.getEntity()).thenReturn(entity); + mockSuccess(response); + + try { + RemoteReindexingUtils.lookupRemoteVersion( + RejectAwareActionListener.wrap( + v -> fail("Expected an exception yet one was not thrown"), + // We're expecting an exception, so no need to fail + e -> {}, + e -> {} + ), + threadPool, + client + ); + } catch (RuntimeException e) { + assertThat(e.getMessage(), containsString("Response didn't include Content-Type: body={")); + } catch (Exception e) { + fail("Expected RuntimeException"); + } + } + + /** + * Verifies that HTTP 429 responses are routed to onRejection rather than onFailure. + */ + public void testLookupRemoteVersionTooManyRequestsTriggersRejection() throws Exception { + AtomicBoolean rejected = new AtomicBoolean(); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(null); + + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(RestStatus.TOO_MANY_REQUESTS.getStatus()); + when(response.getStatusLine()).thenReturn(statusLine); + + // Mocks used in the ResponseException constructor + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("mock"); + when(response.getRequestLine()).thenReturn(requestLine); + mockFailure(new org.elasticsearch.client.ResponseException(response)); + + RemoteReindexingUtils.lookupRemoteVersion( + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> fail("unexpected failure"), e -> rejected.set(true)), + threadPool, + client + ); + assertTrue("onRejection was not called", rejected.get()); + } + + /** + * Verifies that non-429 HTTP errors are routed to onFailure. + */ + public void testLookupRemoteVersionHttpErrorTriggersFailure() throws Exception { + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(RestStatus.BAD_REQUEST.getStatus()); + Response response = mock(Response.class); + when(response.getStatusLine()).thenReturn(statusLine); + when(response.getEntity()).thenReturn(new StringEntity("bad request", ContentType.TEXT_PLAIN)); + + // Mocks used in the ResponseException constructor + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("mock"); + when(response.getRequestLine()).thenReturn(requestLine); + mockFailure(new org.elasticsearch.client.ResponseException(response)); + + RemoteReindexingUtils.lookupRemoteVersion(RejectAwareActionListener.wrap(v -> fail(), ex -> { + assertTrue(ex instanceof ElasticsearchException); + assertEquals(RestStatus.BAD_REQUEST, ((ElasticsearchStatusException) ex).status()); + }, ex -> fail()), threadPool, client); + } + + /** + * Verifies that ContentTooLongException is translated into a user-facing IllegalArgumentException. + */ + public void testContentTooLongExceptionIsWrapped() { + mockFailure(new org.apache.http.ContentTooLongException("too large")); + + RemoteReindexingUtils.lookupRemoteVersion(RejectAwareActionListener.wrap(v -> fail(), ex -> { + assertTrue(ex instanceof IllegalArgumentException); + assertThat(ex.getMessage(), containsString("Remote responded with a chunk that was too large")); + }, ex -> fail()), threadPool, client); + } + + public void testInvalidJsonThrowsElasticsearchException() { + HttpEntity entity = new StringEntity("this is not json", ContentType.APPLICATION_JSON); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + mockSuccess(response); + + RemoteReindexingUtils.lookupRemoteVersion(RejectAwareActionListener.wrap(v -> fail(), ex -> { + assertTrue(ex instanceof ElasticsearchException); + assertThat(ex.getMessage(), containsString("remote is likely not an Elasticsearch instance")); + }, ex -> fail()), threadPool, client); + } + + /** + * Verifies that IOExceptions during response deserialization are surfaced correctly. + */ + public void testIOExceptionDuringDeserialization() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenThrow(new IOException("boom")); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + mockSuccess(response); + + RemoteReindexingUtils.lookupRemoteVersion(RejectAwareActionListener.wrap(v -> fail(), ex -> { + assertTrue(ex instanceof ElasticsearchException); + assertThat(ex.getMessage(), containsString("Error deserializing response")); + }, ex -> fail()), threadPool, client); + } + + public void testWrapExceptionToPreserveStatus() throws IOException { + Exception cause = new Exception(); + + // Successfully get the status without a body + RestStatus status = randomFrom(RestStatus.values()); + ElasticsearchStatusException wrapped = wrapExceptionToPreserveStatus(status.getStatus(), null, cause); + assertEquals(status, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("No error body.", wrapped.getMessage()); + + // Successfully get the status without a body + HttpEntity okEntity = new StringEntity("test body", ContentType.TEXT_PLAIN); + wrapped = wrapExceptionToPreserveStatus(status.getStatus(), okEntity, cause); + assertEquals(status, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("body=test body", wrapped.getMessage()); + + // Successfully get the status with a broken body + IOException badEntityException = new IOException(); + HttpEntity badEntity = mock(HttpEntity.class); + when(badEntity.getContent()).thenThrow(badEntityException); + wrapped = wrapExceptionToPreserveStatus(status.getStatus(), badEntity, cause); + assertEquals(status, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("Failed to extract body.", wrapped.getMessage()); + assertEquals(badEntityException, wrapped.getSuppressed()[0]); + + // Fail to get the status without a body + int notAnHttpStatus = -1; + assertNull(RestStatus.fromCode(notAnHttpStatus)); + wrapped = wrapExceptionToPreserveStatus(notAnHttpStatus, null, cause); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. No error body.", wrapped.getMessage()); + + // Fail to get the status without a body + wrapped = wrapExceptionToPreserveStatus(notAnHttpStatus, okEntity, cause); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. body=test body", wrapped.getMessage()); + + // Fail to get the status with a broken body + wrapped = wrapExceptionToPreserveStatus(notAnHttpStatus, badEntity, cause); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); + assertEquals(cause, wrapped.getCause()); + assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. Failed to extract body.", wrapped.getMessage()); + assertEquals(badEntityException, wrapped.getSuppressed()[0]); + } + + public void testBodyMessageWithNullEntity() throws Exception { + String message = RemoteReindexingUtils.bodyMessage(null); + assertEquals("No error body.", message); + } + + public void testBodyMessageWithReadableEntity() throws Exception { + String testBody = randomAlphanumericOfLength(10); + HttpEntity entity = new StringEntity(testBody, ContentType.TEXT_PLAIN); + + String message = RemoteReindexingUtils.bodyMessage(entity); + + assertEquals("body=" + testBody, message); + } + + public void testBodyMessageWithIOException() throws Exception { + IOException expected = new IOException("Exception"); + + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenThrow(expected); + + IOException actual = expectThrows(IOException.class, () -> RemoteReindexingUtils.bodyMessage(entity)); + + assertSame(expected, actual); + } + + private void mockSuccess(Response response) { + doAnswer(inv -> { + ((org.elasticsearch.client.ResponseListener) inv.getArgument(1)).onSuccess(response); + return null; + }).when(client).performRequestAsync(any(), any()); + } + + private void mockFailure(Exception e) { + doAnswer(inv -> { + ((org.elasticsearch.client.ResponseListener) inv.getArgument(1)).onFailure(e); + return null; + }).when(client).performRequestAsync(any(), any()); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java index 2bff467da58ac..a7b636ed40c8c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.reindex.remote; import org.apache.http.ContentTooLongException; -import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -20,14 +19,12 @@ import org.apache.http.concurrent.FutureCallback; import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer; @@ -56,7 +53,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -67,13 +63,13 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Stream; import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.core.TimeValue.timeValueMinutes; -import static org.hamcrest.Matchers.containsString; +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.execute; +import static org.elasticsearch.reindex.remote.RemoteResponseParsers.RESPONSE_PARSER; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -130,26 +126,6 @@ public void validateAllConsumed() { assertThat(responseQueue, empty()); } - public void testLookupRemoteVersion() throws Exception { - assertLookupRemoteVersion(Version.fromString("0.20.5"), "main/0_20_5.json"); - assertLookupRemoteVersion(Version.fromString("0.90.13"), "main/0_90_13.json"); - assertLookupRemoteVersion(Version.fromString("1.7.5"), "main/1_7_5.json"); - assertLookupRemoteVersion(Version.fromId(2030399), "main/2_3_3.json"); - // assert for V_5_0_0 (no qualifier) since we no longer consider qualifier in Version since 7 - assertLookupRemoteVersion(Version.fromId(5000099), "main/5_0_0_alpha_3.json"); - // V_5_0_0 since we no longer consider qualifier in Version - assertLookupRemoteVersion(Version.fromId(5000099), "main/with_unknown_fields.json"); - } - - private void assertLookupRemoteVersion(Version expected, String s) throws Exception { - AtomicBoolean called = new AtomicBoolean(); - sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, s).lookupRemoteVersion(wrapAsListener(v -> { - assertEquals(expected, v); - called.set(true); - })); - assertTrue(called.get()); - } - public void testParseStartOk() throws Exception { AtomicBoolean called = new AtomicBoolean(); sourceWithMockedRemoteCall("start_ok.json").doStart(wrapAsListener(r -> { @@ -373,55 +349,6 @@ public void testThreadContextRestored() throws Exception { assertTrue(called.get()); } - public void testWrapExceptionToPreserveStatus() throws IOException { - Exception cause = new Exception(); - - // Successfully get the status without a body - RestStatus status = randomFrom(RestStatus.values()); - ElasticsearchStatusException wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(status.getStatus(), null, cause); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("No error body.", wrapped.getMessage()); - - // Successfully get the status without a body - HttpEntity okEntity = new StringEntity("test body", ContentType.TEXT_PLAIN); - wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(status.getStatus(), okEntity, cause); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("body=test body", wrapped.getMessage()); - - // Successfully get the status with a broken body - IOException badEntityException = new IOException(); - HttpEntity badEntity = mock(HttpEntity.class); - when(badEntity.getContent()).thenThrow(badEntityException); - wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(status.getStatus(), badEntity, cause); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Failed to extract body.", wrapped.getMessage()); - assertEquals(badEntityException, wrapped.getSuppressed()[0]); - - // Fail to get the status without a body - int notAnHttpStatus = -1; - assertNull(RestStatus.fromCode(notAnHttpStatus)); - wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, null, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. No error body.", wrapped.getMessage()); - - // Fail to get the status without a body - wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, okEntity, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. body=test body", wrapped.getMessage()); - - // Fail to get the status with a broken body - wrapped = RemoteScrollableHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, badEntity, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. Failed to extract body.", wrapped.getMessage()); - assertEquals(badEntityException, wrapped.getSuppressed()[0]); - } - @SuppressWarnings({ "unchecked", "rawtypes" }) public void testTooLargeResponse() throws Exception { ContentTooLongException tooLong = new ContentTooLongException("too long!"); @@ -454,15 +381,6 @@ public Future answer(InvocationOnMock invocationOnMock) throws Thr assertTrue(responseQueue.isEmpty()); } - public void testNoContentTypeIsError() { - RuntimeException e = expectListenerFailure( - RuntimeException.class, - (RejectAwareActionListener listener) -> sourceWithMockedRemoteCall(false, null, "main/0_20_5.json") - .lookupRemoteVersion(listener) - ); - assertThat(e.getMessage(), containsString("Response didn't include Content-Type: body={")); - } - public void testInvalidJsonThinksRemoteIsNotES() throws Exception { sourceWithMockedRemoteCall("some_text.txt").start(); Throwable e = failureQueue.poll(); @@ -479,7 +397,8 @@ public void testUnexpectedJsonThinksRemoteIsNotES() throws Exception { public void testCleanupSuccessful() throws Exception { AtomicBoolean cleanupCallbackCalled = new AtomicBoolean(); RestClient client = mock(RestClient.class); - TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client); + RemoteInfo remoteInfo = remoteInfo(); + TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client, remoteInfo); hitSource.cleanup(() -> cleanupCallbackCalled.set(true)); verify(client).close(); assertTrue(cleanupCallbackCalled.get()); @@ -489,7 +408,8 @@ public void testCleanupFailure() throws Exception { AtomicBoolean cleanupCallbackCalled = new AtomicBoolean(); RestClient client = mock(RestClient.class); doThrow(new RuntimeException("test")).when(client).close(); - TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client); + RemoteInfo remoteInfo = remoteInfo(); + TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client, remoteInfo); hitSource.cleanup(() -> cleanupCallbackCalled.set(true)); verify(client).close(); assertTrue(cleanupCallbackCalled.get()); @@ -564,13 +484,31 @@ private RemoteScrollableHitSource sourceWithMockedClient(boolean mockRemoteVersi .setHttpClientConfigCallback(httpClientBuilder -> clientBuilder) .build(); - TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(restClient) { + RemoteInfo remoteInfo = remoteInfo(); + TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(restClient, remoteInfo) { + // @Override + // void lookupRemoteVersion(RejectAwareActionListener listener) { + // if (mockRemoteVersion) { + // listener.onResponse(Version.CURRENT); + // } else { + // super.lookupRemoteVersion(listener); + // } + // } + @Override - void lookupRemoteVersion(RejectAwareActionListener listener) { + protected void doStart(RejectAwareActionListener searchListener) { + // Short‑circuit version lookup by setting it to current if (mockRemoteVersion) { - listener.onResponse(Version.CURRENT); + remoteVersion = Version.CURRENT; + execute( + RemoteRequestBuilders.initialSearch(searchRequest, remoteInfo.getQuery(), remoteVersion), + RESPONSE_PARSER, + RejectAwareActionListener.withResponseHandler(searchListener, r -> onStartResponse(searchListener, r)), + threadPool, + restClient + ); } else { - super.lookupRemoteVersion(listener); + super.doStart(searchListener); } } }; @@ -580,6 +518,21 @@ void lookupRemoteVersion(RejectAwareActionListener listener) { return hitSource; } + private RemoteInfo remoteInfo() { + return new RemoteInfo( + "http", + randomAlphaOfLength(8), + randomIntBetween(4000, 9000), + null, + new BytesArray("{}"), + null, + null, + Map.of(), + TimeValue.timeValueSeconds(randomIntBetween(5, 30)), + TimeValue.timeValueSeconds(randomIntBetween(5, 30)) + ); + } + private BackoffPolicy backoff() { return BackoffPolicy.constantBackoff(timeValueMillis(0), retriesAllowed); } @@ -589,7 +542,7 @@ private void countRetry() { } private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource { - TestRemoteScrollableHitSource(RestClient client) { + TestRemoteScrollableHitSource(RestClient client, RemoteInfo remoteInfo) { super( RemoteScrollableHitSourceTests.this.logger, backoff(), @@ -598,18 +551,7 @@ private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource { responseQueue::add, failureQueue::add, client, - new RemoteInfo( - "http", - randomAlphaOfLength(8), - randomIntBetween(4000, 9000), - null, - new BytesArray("{}"), - null, - null, - Map.of(), - TimeValue.timeValueSeconds(randomIntBetween(5, 30)), - TimeValue.timeValueSeconds(randomIntBetween(5, 30)) - ), + remoteInfo, RemoteScrollableHitSourceTests.this.searchRequest ); } @@ -618,15 +560,4 @@ private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource { private RejectAwareActionListener wrapAsListener(Consumer consumer) { return RejectAwareActionListener.wrap(consumer::accept, ESTestCase::fail, ESTestCase::fail); } - - @SuppressWarnings("unchecked") - private T expectListenerFailure(Class expectedException, Consumer> subject) { - AtomicReference exception = new AtomicReference<>(); - subject.accept(RejectAwareActionListener.wrap(r -> fail(), e -> { - assertThat(e, instanceOf(expectedException)); - assertTrue(exception.compareAndSet(null, (T) e)); - }, e -> fail())); - assertNotNull(exception.get()); - return exception.get(); - } } From 66f7debce849d70275af5bdfa7a93e448c141d97 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 13 Feb 2026 16:21:54 +0000 Subject: [PATCH 04/45] Clean up code --- .../org/elasticsearch/reindex/Reindexer.java | 26 ++----------------- .../reindex/TransportReindexAction.java | 1 + .../RemoteScrollableHitSourceTests.java | 9 ------- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index b7465954644bb..aa85e1427bf9a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -82,6 +82,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.synchronizedList; import static org.elasticsearch.index.VersionType.INTERNAL; +import static org.elasticsearch.reindex.TransportReindexAction.REINDEX_PIT_SEARCH_ENABLED; public class Reindexer { @@ -123,7 +124,7 @@ public class Reindexer { public void lookupRemoteVersion(Task task, ReindexRequest request, ActionListener listener) { // If we're reindexing from a remote source, then we need to determine the remote version to decide whether we use // scroll search or point-in-time search - if (request.getRemoteInfo() != null) { + if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { @@ -362,29 +363,6 @@ private IndexMode destinationIndexMode(ProjectState state) { @Override protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy, SearchRequest searchRequest) { - - // TODO: - /* - // We're enabling PIT so make the right decisions - // Done similarly to clusterService.state().clusterFeatures().clusterHasFeature(...) - If feature flag set: - If local request: - Use ClientPittableHitSource - else: - If remote node is PIT compatible: - Use RemotePittableHitSouce - else: - Use RemoteScrollableHitSource - // PIT is not enabled because the cluster is old - else: - // Old logic - */ - - /* - NOTE: By making the PIT decision here we need to refactor very little ... However, we need to open the PIT - in parallisation helper ... so we need to know then ... - */ - if (mainRequest.getRemoteInfo() != null) { RemoteInfo remoteInfo = mainRequest.getRemoteInfo(); createdThreads = synchronizedList(new ArrayList<>()); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index 2be8e4375acb3..1cb05c23082d6 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -45,6 +45,7 @@ public class TransportReindexAction extends HandledTransportAction listener) { - // if (mockRemoteVersion) { - // listener.onResponse(Version.CURRENT); - // } else { - // super.lookupRemoteVersion(listener); - // } - // } - @Override protected void doStart(RejectAwareActionListener searchListener) { // Short‑circuit version lookup by setting it to current From a434f0d1646796a4c9b0cc8bec88ba55a0fb229c Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 13 Feb 2026 16:39:39 +0000 Subject: [PATCH 05/45] Clean up TODOs --- .../src/main/java/org/elasticsearch/reindex/Reindexer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index aa85e1427bf9a..1898b6b6ac6b3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -124,7 +124,7 @@ public class Reindexer { public void lookupRemoteVersion(Task task, ReindexRequest request, ActionListener listener) { // If we're reindexing from a remote source, then we need to determine the remote version to decide whether we use // scroll search or point-in-time search - if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { + if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { @@ -304,7 +304,6 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction()); assert sslConfig != null : "Reindex ssl config must be set"; - // TODO - Remove? Remember, we only use this REST client for remote cases which is orthogonal to whether we use PIT RestClient restClient = buildRestClient(remoteInfo, sslConfig, task.getId(), createdThreads); return new RemoteScrollableHitSource( logger, From 7516540f3924c136e0a29137f4e1b7c38221b4c2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Feb 2026 16:50:26 +0000 Subject: [PATCH 06/45] [CI] Auto commit changes from spotless --- .../src/main/java/org/elasticsearch/reindex/Reindexer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 1898b6b6ac6b3..8ede4a240d255 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -124,7 +124,7 @@ public class Reindexer { public void lookupRemoteVersion(Task task, ReindexRequest request, ActionListener listener) { // If we're reindexing from a remote source, then we need to determine the remote version to decide whether we use // scroll search or point-in-time search - if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { + if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { From 1739e0fc6e6f195f3b478a4a235ed0d392d50dda Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Mon, 23 Feb 2026 14:54:56 +0000 Subject: [PATCH 07/45] Nits --- .../elasticsearch/reindex/remote/RemoteReindexingUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 3d59b99c5025e..d8b835c0040e6 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -34,13 +34,14 @@ import java.io.IOException; import java.io.InputStream; import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.elasticsearch.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER; public class RemoteReindexingUtils { public static void lookupRemoteVersion(RejectAwareActionListener listener, ThreadPool threadPool, RestClient client) { - execute(new Request("GET", ""), MAIN_ACTION_PARSER, listener, threadPool, client); + execute(new Request("GET", "/"), MAIN_ACTION_PARSER, listener, threadPool, client); } static void execute( @@ -51,7 +52,7 @@ static void execute( RestClient client ) { // Preserve the thread context so headers survive after the call - java.util.function.Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); + Supplier contextSupplier = threadPool.getThreadContext().newRestorableContext(true); try { client.performRequestAsync(request, new ResponseListener() { @Override From a805729588310d0d310fb179e356b910b1766eb4 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Mon, 23 Feb 2026 16:49:02 +0000 Subject: [PATCH 08/45] Move remote lookup into execute --- .../BulkByScrollParallelizationHelper.java | 21 ++- .../org/elasticsearch/reindex/Reindexer.java | 151 ++++++++++-------- .../reindex/TransportReindexAction.java | 18 +-- .../remote/RemoteScrollableHitSource.java | 33 +++- .../elasticsearch/reindex/ReindexIdTests.java | 2 +- .../reindex/ReindexMetadataTests.java | 3 +- .../reindex/ReindexScriptTests.java | 3 +- 7 files changed, 136 insertions(+), 95 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java index 0b5be5e0f0b5b..24a5fc205fa96 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java @@ -9,6 +9,7 @@ package org.elasticsearch.reindex; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; @@ -17,6 +18,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; @@ -32,6 +34,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -68,7 +71,9 @@ static > void startSlicedAc task, request, client, - listener.delegateFailure((l, v) -> executeSlicedAction(task, request, action, l, client, node, workerAction)) + listener.delegateFailure( + (l, v) -> executeSlicedAction(task, request, action, l, client, node, null, version -> workerAction.run()) + ) ); } @@ -76,11 +81,14 @@ static > void startSlicedAc * Takes an action and a {@link BulkByScrollTask} and runs it with regard to whether this task is a * leader or worker. * - * If this task is a worker, the worker action in the given {@link Runnable} will be started on the local - * node. If the task is a leader (i.e. the number of slices is more than 1), then a subrequest will be - * created for each slice and sent. + * If this task is a worker, the worker action is invoked with the given {@code remoteVersion} (may be null + * for local reindex). If the task is a leader (i.e. the number of slices is more than 1), then a subrequest + * will be created for each slice and sent. * * This method can only be called after the task state is initialized {@link #initTaskState}. + * + * @param remoteVersion the version of the remote cluster when reindexing from remote, or null for local reindex + * @param workerAction invoked when this task is a worker, with the remote version (or null) */ static > void executeSlicedAction( BulkByScrollTask task, @@ -89,12 +97,13 @@ static > void executeSliced ActionListener listener, Client client, DiscoveryNode node, - Runnable workerAction + @Nullable Version remoteVersion, + Consumer workerAction ) { if (task.isLeader()) { sendSubRequests(client, action, node.getId(), task, request, listener); } else if (task.isWorker()) { - workerAction.run(); + workerAction.accept(remoteVersion); } else { throw new AssertionError("Task should have been initialized at this point."); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 1898b6b6ac6b3..04c12ddb1fa7f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -61,7 +61,6 @@ import org.elasticsearch.script.ReindexScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -77,6 +76,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.LongSupplier; import static java.util.Collections.emptyList; @@ -95,7 +95,6 @@ public class Reindexer { private final ScriptService scriptService; private final ReindexSslConfig reindexSslConfig; private final ReindexMetrics reindexMetrics; - Version remoteVersion; Reindexer( ClusterService clusterService, @@ -115,81 +114,89 @@ public class Reindexer { this.reindexMetrics = reindexMetrics; } - /** - * If we're reindexing from a remote cluster, then we look up the remote version and save it to {@link #remoteVersion} - * Otherwise, we set {@link #remoteVersion} as null - * @param listener The listener to complete once we've determined the remote cluster version. - * This is typically the reindexing request itself. - */ - public void lookupRemoteVersion(Task task, ReindexRequest request, ActionListener listener) { - // If we're reindexing from a remote source, then we need to determine the remote version to decide whether we use - // scroll search or point-in-time search - if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { - RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { - @Override - public void onResponse(Version version) { - remoteVersion = version; - // The listener onResponse will call subsequent reindexing methods. However, the remoteVersion will now be present as a - // class variable rather than a parameter - listener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - public void onRejection(Exception e) { - // No rejection concept in ActionListener, therefore treating as failure - listener.onFailure(e); - } - }; - - RemoteInfo remoteInfo = request.getRemoteInfo(); - assert reindexSslConfig != null : "Reindex ssl config must be set"; - RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); - RemoteReindexingUtils.lookupRemoteVersion(rejectAwareListener, threadPool, restClient); - } - // We're reindexing from the same cluster, so we set the remote version to null - else { - remoteVersion = null; - listener.onResponse(null); - } - } - public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.initTaskState(task, request, client, listener); } public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener listener) { long startTime = System.nanoTime(); + Consumer workerAction = remoteVersion -> { + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); + ParentTaskAssigningClient assigningBulkClient = new ParentTaskAssigningClient(bulkClient, clusterService.localNode(), task); + AsyncIndexBySearchAction searchAction = new AsyncIndexBySearchAction( + task, + logger, + assigningClient, + assigningBulkClient, + threadPool, + scriptService, + projectResolver.getProjectState(clusterService.state()), + reindexSslConfig, + request, + wrapWithMetrics(listener, reindexMetrics, startTime, request.getRemoteInfo() != null), + remoteVersion + ); + searchAction.start(); + }; + + /** + * If this is a request to reindex from remote, then we need to determine the remote version prior to execution + * NB {@link ReindexRequest} forbids remote requests and slices > 1, so we're guaranteed to be running on the only slice + */ + if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { + lookupRemoteVersionAndExecute(task, request, listener, workerAction); + } else { + BulkByScrollParallelizationHelper.executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + clusterService.localNode(), + null, + workerAction + ); + } + } - BulkByScrollParallelizationHelper.executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - clusterService.localNode(), - () -> { - ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); - ParentTaskAssigningClient assigningBulkClient = new ParentTaskAssigningClient(bulkClient, clusterService.localNode(), task); - AsyncIndexBySearchAction searchAction = new AsyncIndexBySearchAction( + /** + * Looks up the remote cluster version when reindexing from a remote source, then runs the sliced action with that version. + */ + private void lookupRemoteVersionAndExecute( + BulkByScrollTask task, + ReindexRequest request, + ActionListener listener, + Consumer workerAction + ) { + RemoteInfo remoteInfo = request.getRemoteInfo(); + assert reindexSslConfig != null : "Reindex ssl config must be set"; + RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); + RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { + @Override + public void onResponse(Version version) { + BulkByScrollParallelizationHelper.executeSlicedAction( task, - logger, - assigningClient, - assigningBulkClient, - threadPool, - scriptService, - projectResolver.getProjectState(clusterService.state()), - reindexSslConfig, request, - wrapWithMetrics(listener, reindexMetrics, startTime, request.getRemoteInfo() != null) + ReindexAction.INSTANCE, + listener, + client, + clusterService.localNode(), + version, + workerAction ); - searchAction.start(); } - ); + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + public void onRejection(Exception e) { + listener.onFailure(e); + } + }; + RemoteReindexingUtils.lookupRemoteVersion(rejectAwareListener, threadPool, restClient); } // Visible for testing @@ -312,6 +319,11 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction createdThreads = emptyList(); + /** + * Version of the remote cluster when reindexing from remote, or null when reindexing locally. + */ + private final Version remoteVersion; + AsyncIndexBySearchAction( BulkByScrollTask task, Logger logger, @@ -322,7 +334,8 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener + ActionListener listener, + @Nullable Version remoteVersion ) { super( task, @@ -343,6 +356,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener) { validate(request); BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; - - /* - 1. If we're reindexing from a remote cluster, then the cluster version is obtained - 2. We initialise the reindexing task - 3. The reindexing task is executed. - */ - reindexer.lookupRemoteVersion( - task, + reindexer.initTask( + bulkByScrollTask, request, - listener.delegateFailure( - (l, v) -> reindexer.initTask( - bulkByScrollTask, - request, - l.delegateFailure((l2, v2) -> reindexer.execute(bulkByScrollTask, request, getBulkClient(), l2)) - ) - ) + listener.delegateFailure((l, v) -> reindexer.execute(bulkByScrollTask, request, getBulkClient(), l)) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java index 8ba2985d8b3ce..a75523f19b981 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java @@ -18,6 +18,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RemoteInfo; @@ -52,17 +53,32 @@ public RemoteScrollableHitSource( RestClient client, RemoteInfo remoteInfo, SearchRequest searchRequest + ) { + this(logger, backoffPolicy, threadPool, countSearchRetry, onResponse, fail, client, remoteInfo, searchRequest, null); + } + + public RemoteScrollableHitSource( + Logger logger, + BackoffPolicy backoffPolicy, + ThreadPool threadPool, + Runnable countSearchRetry, + Consumer onResponse, + Consumer fail, + RestClient client, + RemoteInfo remoteInfo, + SearchRequest searchRequest, + @Nullable Version initialRemoteVersion ) { super(logger, backoffPolicy, threadPool, countSearchRetry, onResponse, fail); this.remote = remoteInfo; this.searchRequest = searchRequest; this.client = client; + this.remoteVersion = initialRemoteVersion; } @Override protected void doStart(RejectAwareActionListener searchListener) { - lookupRemoteVersion(RejectAwareActionListener.withResponseHandler(searchListener, version -> { - remoteVersion = version; + if (remoteVersion != null) { execute( RemoteRequestBuilders.initialSearch(searchRequest, remote.getQuery(), remoteVersion), RESPONSE_PARSER, @@ -70,7 +86,18 @@ protected void doStart(RejectAwareActionListener searchListener) { threadPool, client ); - }), threadPool, client); + } else { + lookupRemoteVersion(RejectAwareActionListener.withResponseHandler(searchListener, version -> { + remoteVersion = version; + execute( + RemoteRequestBuilders.initialSearch(searchRequest, remote.getQuery(), remoteVersion), + RESPONSE_PARSER, + RejectAwareActionListener.withResponseHandler(searchListener, r -> onStartResponse(searchListener, r)), + threadPool, + client + ); + }), threadPool, client); + } } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java index 1a6b835a7ce69..2ec5af871636f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java @@ -109,6 +109,6 @@ protected ReindexRequest request() { } private Reindexer.AsyncIndexBySearchAction action(ProjectState state) { - return new Reindexer.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, state, null, request(), listener()); + return new Reindexer.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, state, null, request(), listener(), null); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java index 30ae5541fad42..9229d671307d8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java @@ -82,7 +82,8 @@ private class TestAction extends Reindexer.AsyncIndexBySearchAction { ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID), null, request(), - listener() + listener(), + null ); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java index b95896bd0298f..12400ebd5dc60 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java @@ -101,7 +101,8 @@ protected Reindexer.AsyncIndexBySearchAction action(ScriptService scriptService, ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID), sslConfig, request, - listener() + listener(), + null ); } } From d5471f07074bdb05c27beffd19d56affb038845b Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Mon, 23 Feb 2026 17:02:40 +0000 Subject: [PATCH 09/45] Add Tests --- ...ulkByScrollParallelizationHelperTests.java | 118 ++++++++++++++++++ .../RemoteScrollableHitSourceTests.java | 97 ++++++++++++++ 2 files changed, 215 insertions(+) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java index ebb4471566fbd..c0d35ac329db5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java @@ -9,19 +9,54 @@ package org.elasticsearch.reindex; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; +import static java.util.Collections.emptySet; +import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.executeSlicedAction; import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertThat; public class BulkByScrollParallelizationHelperTests extends ESTestCase { + + private ThreadPool threadPool; + private TaskManager taskManager; + + @Before + public void setUpTaskManager() { + threadPool = new TestThreadPool(getTestName()); + taskManager = new TaskManager(Settings.EMPTY, threadPool, emptySet()); + } + + @After + public void tearDownTaskManager() { + terminate(threadPool); + } public void testSliceIntoSubRequests() throws IOException { SearchRequest searchRequest = randomSearchRequest( () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null) @@ -48,4 +83,87 @@ public void testSliceIntoSubRequests() throws IOException { currentSliceId++; } } + + /** + * When the task is a worker, executeSlicedAction invokes the worker action with the given remote version. + */ + public void testExecuteSlicedActionWithWorkerAndNonNullVersion() { + ReindexRequest request = new ReindexRequest(); + BulkByScrollTask task = (BulkByScrollTask) taskManager.register("reindex", ReindexAction.NAME, request); + task.setWorker(request.getRequestsPerSecond(), null); + + Version version = Version.CURRENT; + AtomicReference capturedVersion = new AtomicReference<>(); + ActionListener listener = ActionListener.noop(); + Client client = null; + DiscoveryNode node = DiscoveryNodeUtils.builder("node").roles(emptySet()).build(); + + executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + node, + version, + capturedVersion::set + ); + + assertThat(capturedVersion.get(), sameInstance(version)); + } + + /** + * When the task is a worker and remote version is null (local reindex), the worker action receives null. + */ + public void testExecuteSlicedActionWithWorkerAndNullVersion() { + ReindexRequest request = new ReindexRequest(); + BulkByScrollTask task = (BulkByScrollTask) taskManager.register("reindex", ReindexAction.NAME, request); + task.setWorker(request.getRequestsPerSecond(), null); + + AtomicReference capturedVersion = new AtomicReference<>(Version.CURRENT); + ActionListener listener = ActionListener.noop(); + Client client = null; + DiscoveryNode node = DiscoveryNodeUtils.builder("node").roles(emptySet()).build(); + + executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + node, + null, + capturedVersion::set + ); + + assertThat(capturedVersion.get(), nullValue()); + } + + /** + * When the task is neither a leader nor a worker (not initialized), executeSlicedAction throws. + */ + public void testExecuteSlicedActionThrowsWhenTaskNotInitialized() { + ReindexRequest request = new ReindexRequest(); + BulkByScrollTask task = (BulkByScrollTask) taskManager.register("reindex", ReindexAction.NAME, request); + // Do not call setWorker or setWorkerCount + + ActionListener listener = ActionListener.noop(); + Client client = null; + DiscoveryNode node = DiscoveryNodeUtils.builder("node").roles(emptySet()).build(); + + AssertionError e = expectThrows( + AssertionError.class, + () -> executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + node, + null, + v -> {} + ) + ); + assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("initialized")); + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java index 984175b91822e..9fbdbd7741f6c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java @@ -276,6 +276,23 @@ public void testParseFailureWithStatus() throws Exception { assertTrue(called.get()); } + /** + * When constructed with a non-null initial remote version, doStart skips the version lookup and issues + * the initial search directly. Only one HTTP request is made (the search), not two (version + search). + */ + public void testDoStartSkipsVersionLookupWhenInitialRemoteVersionSet() throws Exception { + AtomicBoolean called = new AtomicBoolean(); + RemoteScrollableHitSource hitSource = sourceWithInitialRemoteVersion(Version.CURRENT, "start_ok.json"); + hitSource.doStart(wrapAsListener(r -> { + assertFalse(r.isTimedOut()); + assertEquals(FAKE_SCROLL_ID, r.getScrollId()); + assertEquals(4, r.getTotalHits()); + assertThat(r.getHits(), hasSize(1)); + called.set(true); + })); + assertTrue(called.get()); + } + public void testParseRequestFailure() throws Exception { AtomicBoolean called = new AtomicBoolean(); Consumer checkResponse = r -> { @@ -509,6 +526,86 @@ protected void doStart(RejectAwareActionListener searchListener) { return hitSource; } + /** + * Creates a RemoteScrollableHitSource with a pre-resolved initial remote version so that doStart skips the version lookup. + * The mock client serves only the given response paths (one request = one path when using initial version). + */ + private RemoteScrollableHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) throws Exception { + return sourceWithInitialRemoteVersion(initialRemoteVersion, ContentType.APPLICATION_JSON, paths); + } + + @SuppressWarnings("unchecked") + private RemoteScrollableHitSource sourceWithInitialRemoteVersion( + Version initialRemoteVersion, + ContentType contentType, + String... paths + ) throws Exception { + URL[] resources = new URL[paths.length]; + for (int i = 0; i < paths.length; i++) { + resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", "")); + if (resources[i] == null) { + throw new IllegalArgumentException("Couldn't find [" + paths[i] + "]"); + } + } + + CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class); + when( + httpClient.execute( + any(HttpAsyncRequestProducer.class), + any(HttpAsyncResponseConsumer.class), + any(HttpClientContext.class), + any(FutureCallback.class) + ) + ).thenAnswer(new Answer>() { + int responseCount = 0; + + @Override + public Future answer(InvocationOnMock invocationOnMock) throws Throwable { + threadPool.getThreadContext().stashContext(); + FutureCallback futureCallback = (FutureCallback) invocationOnMock.getArguments()[3]; + HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; + HttpEntityEnclosingRequest request = (HttpEntityEnclosingRequest) requestProducer.generateRequest(); + URL resource = resources[responseCount]; + String path = paths[responseCount++]; + ProtocolVersion protocolVersion = new ProtocolVersion("http", 1, 1); + if (path.startsWith("fail:")) { + String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); + if (path.equals("fail:rejection.json")) { + StatusLine statusLine = new BasicStatusLine(protocolVersion, RestStatus.TOO_MANY_REQUESTS.getStatus(), ""); + futureCallback.completed(new BasicHttpResponse(statusLine)); + } else { + futureCallback.failed(new RuntimeException(body)); + } + } else { + StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, ""); + HttpResponse httpResponse = new BasicHttpResponse(statusLine); + httpResponse.setEntity(new InputStreamEntity(FileSystemUtils.openFileURLStream(resource), contentType)); + futureCallback.completed(httpResponse); + } + return null; + } + }); + + HttpAsyncClientBuilder clientBuilder = mock(HttpAsyncClientBuilder.class); + when(clientBuilder.build()).thenReturn(httpClient); + RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)) + .setHttpClientConfigCallback(httpClientBuilder -> clientBuilder) + .build(); + + return new RemoteScrollableHitSource( + logger, + backoff(), + threadPool, + this::countRetry, + responseQueue::add, + failureQueue::add, + restClient, + remoteInfo(), + searchRequest, + initialRemoteVersion + ); + } + private RemoteInfo remoteInfo() { return new RemoteInfo( "http", From 6b86337736ebfba346af91b73e5cf1cfec5e8ae5 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Mon, 23 Feb 2026 17:20:31 +0000 Subject: [PATCH 10/45] Merge conflicts --- .../org/elasticsearch/reindex/Reindexer.java | 2 +- .../RemoteScrollablePaginatedHitSource.java | 1 - ...natedSearchParallelizationHelperTests.java | 37 ++------- ...moteScrollablePaginatedHitSourceTests.java | 77 ++++--------------- 4 files changed, 19 insertions(+), 98 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index b644a9cdf3780..c4bbae338fe5e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -54,8 +54,8 @@ import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState; -import org.elasticsearch.reindex.remote.RemoteScrollablePaginatedHitSource; import org.elasticsearch.reindex.remote.RemoteReindexingUtils; +import org.elasticsearch.reindex.remote.RemoteScrollablePaginatedHitSource; import org.elasticsearch.script.CtxMap; import org.elasticsearch.script.ReindexMetadata; import org.elasticsearch.script.ReindexScript; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java index 0d55c937c68c6..cf519780c69f4 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.ResumeInfo.ScrollWorkerResumeInfo; import org.elasticsearch.index.reindex.ResumeInfo.WorkerResumeInfo; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index 2a34f0cc98d45..e671acb56f84f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -16,13 +16,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -99,16 +99,7 @@ public void testExecuteSlicedActionWithWorkerAndNonNullVersion() { Client client = null; DiscoveryNode node = DiscoveryNodeUtils.builder("node").roles(emptySet()).build(); - executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - node, - version, - capturedVersion::set - ); + executeSlicedAction(task, request, ReindexAction.INSTANCE, listener, client, node, version, capturedVersion::set); assertThat(capturedVersion.get(), sameInstance(version)); } @@ -126,16 +117,7 @@ public void testExecuteSlicedActionWithWorkerAndNullVersion() { Client client = null; DiscoveryNode node = DiscoveryNodeUtils.builder("node").roles(emptySet()).build(); - executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - node, - null, - capturedVersion::set - ); + executeSlicedAction(task, request, ReindexAction.INSTANCE, listener, client, node, null, capturedVersion::set); assertThat(capturedVersion.get(), nullValue()); } @@ -154,16 +136,7 @@ public void testExecuteSlicedActionThrowsWhenTaskNotInitialized() { AssertionError e = expectThrows( AssertionError.class, - () -> executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - node, - null, - v -> {} - ) + () -> executeSlicedAction(task, request, ReindexAction.INSTANCE, listener, client, node, null, v -> {}) ); assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("initialized")); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java index a5b41f1b39a10..ab8159cbd43c9 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.reindex.remote; import org.apache.http.ContentTooLongException; +import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -19,12 +20,14 @@ import org.apache.http.concurrent.FutureCallback; import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer; @@ -53,6 +56,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -282,7 +286,7 @@ public void testParseFailureWithStatus() throws Exception { */ public void testDoStartSkipsVersionLookupWhenInitialRemoteVersionSet() throws Exception { AtomicBoolean called = new AtomicBoolean(); - RemoteScrollableHitSource hitSource = sourceWithInitialRemoteVersion(Version.CURRENT, "start_ok.json"); + RemoteScrollablePaginatedHitSource hitSource = sourceWithInitialRemoteVersion(Version.CURRENT, "start_ok.json"); hitSource.doStart(wrapAsListener(r -> { assertFalse(r.isTimedOut()); assertEquals(FAKE_SCROLL_ID, r.getScrollId()); @@ -366,59 +370,6 @@ public void testThreadContextRestored() throws Exception { assertTrue(called.get()); } - public void testWrapExceptionToPreserveStatus() throws IOException { - Exception cause = new Exception(); - - // Successfully get the status without a body - RestStatus status = randomFrom(RestStatus.values()); - ElasticsearchStatusException wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus( - status.getStatus(), - null, - cause - ); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("No error body.", wrapped.getMessage()); - - // Successfully get the status without a body - HttpEntity okEntity = new StringEntity("test body", ContentType.TEXT_PLAIN); - wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus(status.getStatus(), okEntity, cause); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("body=test body", wrapped.getMessage()); - - // Successfully get the status with a broken body - IOException badEntityException = new IOException(); - HttpEntity badEntity = mock(HttpEntity.class); - when(badEntity.getContent()).thenThrow(badEntityException); - wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus(status.getStatus(), badEntity, cause); - assertEquals(status, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Failed to extract body.", wrapped.getMessage()); - assertEquals(badEntityException, wrapped.getSuppressed()[0]); - - // Fail to get the status without a body - int notAnHttpStatus = -1; - assertNull(RestStatus.fromCode(notAnHttpStatus)); - wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, null, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. No error body.", wrapped.getMessage()); - - // Fail to get the status without a body - wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, okEntity, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. body=test body", wrapped.getMessage()); - - // Fail to get the status with a broken body - wrapped = RemoteScrollablePaginatedHitSource.wrapExceptionToPreserveStatus(notAnHttpStatus, badEntity, cause); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, wrapped.status()); - assertEquals(cause, wrapped.getCause()); - assertEquals("Couldn't extract status [" + notAnHttpStatus + "]. Failed to extract body.", wrapped.getMessage()); - assertEquals(badEntityException, wrapped.getSuppressed()[0]); - } - @SuppressWarnings({ "unchecked", "rawtypes" }) public void testTooLargeResponse() throws Exception { ContentTooLongException tooLong = new ContentTooLongException("too long!"); @@ -468,7 +419,7 @@ public void testCleanupSuccessful() throws Exception { AtomicBoolean cleanupCallbackCalled = new AtomicBoolean(); RestClient client = mock(RestClient.class); RemoteInfo remoteInfo = remoteInfo(); - TestRemoteScrollablePaginatedHitSource paginatedHitSource = new TestRemoteScrollableHitSource(client, remoteInfo); + TestRemoteScrollablePaginatedHitSource paginatedHitSource = new TestRemoteScrollablePaginatedHitSource(client, remoteInfo); paginatedHitSource.cleanup(() -> cleanupCallbackCalled.set(true)); verify(client).close(); assertTrue(cleanupCallbackCalled.get()); @@ -479,7 +430,7 @@ public void testCleanupFailure() throws Exception { RestClient client = mock(RestClient.class); doThrow(new RuntimeException("test")).when(client).close(); RemoteInfo remoteInfo = remoteInfo(); - TestRemoteScrollablePaginatedHitSource paginatedHitSource = new TestRemoteScrollableHitSource(client, remoteInfo); + TestRemoteScrollablePaginatedHitSource paginatedHitSource = new TestRemoteScrollablePaginatedHitSource(client, remoteInfo); paginatedHitSource.cleanup(() -> cleanupCallbackCalled.set(true)); verify(client).close(); assertTrue(cleanupCallbackCalled.get()); @@ -584,19 +535,16 @@ protected void doStart(RejectAwareActionListener searchListener) { } /** - * Creates a RemoteScrollableHitSource with a pre-resolved initial remote version so that doStart skips the version lookup. + * Creates a RemoteScrollablePaginatedHitSource with a pre-resolved initial remote version so that doStart skips the version lookup. * The mock client serves only the given response paths (one request = one path when using initial version). */ - private RemoteScrollableHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) throws Exception { return sourceWithInitialRemoteVersion(initialRemoteVersion, ContentType.APPLICATION_JSON, paths); } @SuppressWarnings("unchecked") - private RemoteScrollableHitSource sourceWithInitialRemoteVersion( - Version initialRemoteVersion, - ContentType contentType, - String... paths - ) throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, ContentType contentType, String... paths) + throws Exception { URL[] resources = new URL[paths.length]; for (int i = 0; i < paths.length; i++) { resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", "")); @@ -649,7 +597,7 @@ public Future answer(InvocationOnMock invocationOnMock) throws Thr .setHttpClientConfigCallback(httpClientBuilder -> clientBuilder) .build(); - return new RemoteScrollableHitSource( + return new RemoteScrollablePaginatedHitSource( logger, backoff(), threadPool, @@ -712,6 +660,7 @@ private class TestRemoteScrollablePaginatedHitSource extends RemoteScrollablePag ); } } + private RejectAwareActionListener wrapAsListener(Consumer consumer) { return RejectAwareActionListener.wrap(consumer::accept, ESTestCase::fail, ESTestCase::fail); } From 13bdc0b7ca3dd89ed32ccec75d5e6948c9a8eeee Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 23 Feb 2026 17:31:26 +0000 Subject: [PATCH 11/45] [CI] Auto commit changes from spotless --- .../RemoteScrollablePaginatedHitSourceTests.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java index ab8159cbd43c9..a6ae879764ead 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.reindex.remote; import org.apache.http.ContentTooLongException; -import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -20,14 +19,12 @@ import org.apache.http.concurrent.FutureCallback; import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer; @@ -56,7 +53,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -538,13 +534,17 @@ protected void doStart(RejectAwareActionListener searchListener) { * Creates a RemoteScrollablePaginatedHitSource with a pre-resolved initial remote version so that doStart skips the version lookup. * The mock client serves only the given response paths (one request = one path when using initial version). */ - private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) + throws Exception { return sourceWithInitialRemoteVersion(initialRemoteVersion, ContentType.APPLICATION_JSON, paths); } @SuppressWarnings("unchecked") - private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, ContentType contentType, String... paths) - throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion( + Version initialRemoteVersion, + ContentType contentType, + String... paths + ) throws Exception { URL[] resources = new URL[paths.length]; for (int i = 0; i < paths.length; i++) { resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", "")); From 1dd0afc4d1f8dc8607459feb64e3a3fa58b8200c Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Mon, 23 Feb 2026 17:34:35 +0000 Subject: [PATCH 12/45] Clean up --- .../org/elasticsearch/reindex/Reindexer.java | 2 +- .../reindex/TransportReindexAction.java | 8 ----- .../elasticsearch/reindex/ReindexIdTests.java | 3 +- .../reindex/ReindexMetadataTests.java | 3 +- .../reindex/ReindexScriptTests.java | 3 +- ...moteScrollablePaginatedHitSourceTests.java | 31 ++++++------------- 6 files changed, 17 insertions(+), 33 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index c4bbae338fe5e..f1efa0d49d5f9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -82,7 +82,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.synchronizedList; import static org.elasticsearch.index.VersionType.INTERNAL; -import static org.elasticsearch.reindex.TransportReindexAction.REINDEX_PIT_SEARCH_ENABLED; +import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; public class Reindexer { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index dbc243b8fc1fc..bc8acfd476c5b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -41,13 +40,6 @@ public class TransportReindexAction extends HandledTransportAction searchListener) { // Short‑circuit version lookup by setting it to current @@ -538,13 +534,17 @@ protected void doStart(RejectAwareActionListener searchListener) { * Creates a RemoteScrollablePaginatedHitSource with a pre-resolved initial remote version so that doStart skips the version lookup. * The mock client serves only the given response paths (one request = one path when using initial version). */ - private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, String... paths) + throws Exception { return sourceWithInitialRemoteVersion(initialRemoteVersion, ContentType.APPLICATION_JSON, paths); } @SuppressWarnings("unchecked") - private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion(Version initialRemoteVersion, ContentType contentType, String... paths) - throws Exception { + private RemoteScrollablePaginatedHitSource sourceWithInitialRemoteVersion( + Version initialRemoteVersion, + ContentType contentType, + String... paths + ) throws Exception { URL[] resources = new URL[paths.length]; for (int i = 0; i < paths.length; i++) { resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", "")); @@ -635,7 +635,7 @@ private void countRetry() { } private class TestRemoteScrollablePaginatedHitSource extends RemoteScrollablePaginatedHitSource { - TestRemoteScrollablePaginatedHitSource(RestClient client) { + TestRemoteScrollablePaginatedHitSource(RestClient client, RemoteInfo remoteInfo) { super( RemoteScrollablePaginatedHitSourceTests.this.logger, backoff(), @@ -644,18 +644,7 @@ private class TestRemoteScrollablePaginatedHitSource extends RemoteScrollablePag responseQueue::add, failureQueue::add, client, - new RemoteInfo( - "http", - randomAlphaOfLength(8), - randomIntBetween(4000, 9000), - null, - new BytesArray("{}"), - null, - null, - Map.of(), - TimeValue.timeValueSeconds(randomIntBetween(5, 30)), - TimeValue.timeValueSeconds(randomIntBetween(5, 30)) - ), + remoteInfo, RemoteScrollablePaginatedHitSourceTests.this.searchRequest ); } From 460140c3dd6b5a94352b515e4b1f98429f8a919a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 23 Feb 2026 17:40:15 +0000 Subject: [PATCH 13/45] [CI] Auto commit changes from spotless --- .../org/elasticsearch/reindex/ReindexIdTests.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java index 88e71c41e1854..dbdff2c0a74b4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java @@ -110,6 +110,18 @@ protected ReindexRequest request() { } private Reindexer.AsyncIndexBySearchAction action(ProjectState state) { - return new Reindexer.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, state, null, request(), listener(), randomBoolean() ? null : Version.CURRENT); + return new Reindexer.AsyncIndexBySearchAction( + task, + logger, + null, + null, + threadPool, + null, + state, + null, + request(), + listener(), + randomBoolean() ? null : Version.CURRENT + ); } } From 5fd8ab061c033b1dde85fceb328894bf094d3afb Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Tue, 24 Feb 2026 12:15:28 +0000 Subject: [PATCH 14/45] Update RemoteScrollablePaginatedHitSourceTests to randomly set remote version --- .../remote/RemoteScrollablePaginatedHitSourceTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java index ff2d4d303dc54..90b748767bf8f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSourceTests.java @@ -645,7 +645,8 @@ private class TestRemoteScrollablePaginatedHitSource extends RemoteScrollablePag failureQueue::add, client, remoteInfo, - RemoteScrollablePaginatedHitSourceTests.this.searchRequest + RemoteScrollablePaginatedHitSourceTests.this.searchRequest, + randomBoolean() ? Version.CURRENT : null ); } } From ec2517bf7c4069a567f24a144a4bf71731308173 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Tue, 24 Feb 2026 12:27:15 +0000 Subject: [PATCH 15/45] Close REST client --- .../org/elasticsearch/reindex/Reindexer.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index f1efa0d49d5f9..50f01985096a4 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -161,6 +161,8 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl /** * Looks up the remote cluster version when reindexing from a remote source, then runs the sliced action with that version. + * The RestClient used for the lookup is closed after the callback; closing must happen on a thread other than the + * RestClient's own thread pool to avoid shutdown failures. */ private void lookupRemoteVersionAndExecute( BulkByScrollTask task, @@ -174,7 +176,7 @@ private void lookupRemoteVersionAndExecute( RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { - BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( + closeRestClientAndRun(restClient, () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( task, request, ReindexAction.INSTANCE, @@ -183,22 +185,37 @@ public void onResponse(Version version) { clusterService.localNode(), version, workerAction - ); + )); } @Override public void onFailure(Exception e) { - listener.onFailure(e); + closeRestClientAndRun(restClient, () -> listener.onFailure(e)); } @Override public void onRejection(Exception e) { - listener.onFailure(e); + closeRestClientAndRun(restClient, () -> listener.onFailure(e)); } }; RemoteReindexingUtils.lookupRemoteVersion(rejectAwareListener, threadPool, restClient); } + /** + * Closes the RestClient on the generic thread pool (to avoid closing from the client's own thread), then runs the given action. + */ + private void closeRestClientAndRun(RestClient restClient, Runnable onCompletion) { + threadPool.generic().submit(() -> { + try { + restClient.close(); + } catch (IOException e) { + logger.warn("Failed to close RestClient after version lookup", e); + } finally { + onCompletion.run(); + } + }); + } + // Visible for testing static ActionListener wrapWithMetrics( ActionListener listener, From 3ba167083281d4330701b7d2b21e867ce4379811 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 24 Feb 2026 12:34:09 +0000 Subject: [PATCH 16/45] [CI] Auto commit changes from spotless --- .../org/elasticsearch/reindex/Reindexer.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 50f01985096a4..afdccb8672051 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -176,16 +176,19 @@ private void lookupRemoteVersionAndExecute( RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { - closeRestClientAndRun(restClient, () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - clusterService.localNode(), - version, - workerAction - )); + closeRestClientAndRun( + restClient, + () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + clusterService.localNode(), + version, + workerAction + ) + ); } @Override From 363f2c27726f3466f46d451834fee029699742f0 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 11:22:10 +0000 Subject: [PATCH 17/45] Add retry logic to remote version lookup --- .../org/elasticsearch/reindex/Reindexer.java | 33 ++++++---- .../reindex/remote/RemoteReindexingUtils.java | 64 ++++++++++++++++++- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 50f01985096a4..29e93ac193bf2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -81,6 +81,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.synchronizedList; +import static org.elasticsearch.common.BackoffPolicy.exponentialBackoff; import static org.elasticsearch.index.VersionType.INTERNAL; import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; @@ -176,16 +177,19 @@ private void lookupRemoteVersionAndExecute( RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { - closeRestClientAndRun(restClient, () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listener, - client, - clusterService.localNode(), - version, - workerAction - )); + closeRestClientAndRun( + restClient, + () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + clusterService.localNode(), + version, + workerAction + ) + ); } @Override @@ -198,7 +202,14 @@ public void onRejection(Exception e) { closeRestClientAndRun(restClient, () -> listener.onFailure(e)); } }; - RemoteReindexingUtils.lookupRemoteVersion(rejectAwareListener, threadPool, restClient); + RemoteReindexingUtils.lookupRemoteVersionWithRetries( + logger, + exponentialBackoff(request.getRetryBackoffInitialTime(), request.getMaxRetries()), + threadPool, + restClient, + task.getWorkerState()::countSearchRetry, + rejectAwareListener + ); } /** diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index d8b835c0040e6..36abf08337f96 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -13,6 +13,7 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; @@ -20,10 +21,12 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.RejectAwareActionListener; +import org.elasticsearch.index.reindex.RetryListener; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentParseException; @@ -38,12 +41,62 @@ import static org.elasticsearch.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER; +/** + * Utility methods for reindexing from remote Elasticsearch clusters. + * Handles version lookup, HTTP execution with response parsing, and error handling. + */ public class RemoteReindexingUtils { + /** + * Looks up the version of the remote Elasticsearch cluster by performing a GET request to the root path. + * + * @param listener receives the parsed version on success, or failure/rejection on error + * @param threadPool thread pool for preserving thread context across async callbacks + * @param client REST client for the remote cluster + */ public static void lookupRemoteVersion(RejectAwareActionListener listener, ThreadPool threadPool, RestClient client) { execute(new Request("GET", "/"), MAIN_ACTION_PARSER, listener, threadPool, client); } + /** + * Looks up the remote cluster version with retries on rejection (e.g. 429 Too Many Requests). + * Matches the retry behavior used by {@link RemoteScrollablePaginatedHitSource} when it looks up the version. + * + * @param logger logger for retry messages + * @param backoffPolicy policy for delay between retries + * @param threadPool thread pool for scheduling retries + * @param client REST client for the remote cluster + * @param countRetry invoked on each retry attempt + * @param delegate receives the version on success or failure after all retries exhausted + */ + public static void lookupRemoteVersionWithRetries( + Logger logger, + BackoffPolicy backoffPolicy, + ThreadPool threadPool, + RestClient client, + Runnable countRetry, + RejectAwareActionListener delegate + ) { + RetryListener retryListener = new RetryListener<>(logger, threadPool, backoffPolicy, listener -> { + countRetry.run(); + lookupRemoteVersion(listener, threadPool, client); + }, delegate); + lookupRemoteVersion(retryListener, threadPool, client); + } + + /** + * Performs an async HTTP request to the remote cluster, parses the response, and notifies the listener. + * Preserves thread context across the async callback. On 429 (Too Many Requests), invokes + * {@link RejectAwareActionListener#onRejection} so callers can retry; other failures invoke + * {@link RejectAwareActionListener#onFailure}. + * + * @param type of the parsed response + * @param request HTTP request to perform + * @param parser function to parse the response body into type T + * @param listener receives the parsed result, or failure/rejection + * @param threadPool thread pool for preserving thread context + * @param client REST client for the remote cluster + */ static void execute( Request request, BiFunction parser, @@ -134,8 +187,8 @@ public void onFailure(Exception e) { } /** - * Wrap the ResponseException in an exception that'll preserve its status code if possible so we can send it back to the user. We might - * not have a constant for the status code so in that case we just use 500 instead. We also extract make sure to include the response + * Wrap the ResponseException in an exception that'll preserve its status code if possible, so we can send it back to the user. We might + * not have a constant for the status code, so in that case, we just use 500 instead. We also extract make sure to include the response * body in the message so the user can figure out *why* the remote Elasticsearch service threw the error back to us. */ static ElasticsearchStatusException wrapExceptionToPreserveStatus(int statusCode, @Nullable HttpEntity entity, Exception cause) { @@ -154,6 +207,13 @@ static ElasticsearchStatusException wrapExceptionToPreserveStatus(int statusCode } } + /** + * Extracts a readable string from an HTTP entity for use in error messages. + * + * @param entity HTTP entity, or null + * @return "No error body." if entity is null, otherwise "body=" + entity content + * @throws IOException if reading the entity fails + */ static String bodyMessage(@Nullable HttpEntity entity) throws IOException { if (entity == null) { return "No error body."; From 2a4e99961019e463123ae7741445d2bd36584310 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 11:33:26 +0000 Subject: [PATCH 18/45] Add retry logic unit tests --- .../remote/RemoteReindexingUtilsTests.java | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 5eaf4d4f1a8d8..cd677a9d92ab9 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -14,16 +14,21 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -31,17 +36,24 @@ import java.io.IOException; import java.net.URL; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.wrapExceptionToPreserveStatus; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class RemoteReindexingUtilsTests extends ESTestCase { + + private static final Logger logger = LogManager.getLogger(RemoteReindexingUtilsTests.class); + private ThreadPool threadPool; private RestClient client; @@ -53,6 +65,12 @@ public void setUp() throws Exception { public ExecutorService executor(String name) { return EsExecutors.DIRECT_EXECUTOR_SERVICE; } + + @Override + public Scheduler.ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor executor) { + command.run(); + return null; + } }; client = mock(RestClient.class); } @@ -291,6 +309,167 @@ public void testBodyMessageWithIOException() throws Exception { assertSame(expected, actual); } + /** + * Verifies that lookupRemoteVersionWithRetries retries on 429 and eventually succeeds. + */ + public void testLookupRemoteVersionWithRetriesSucceedsOnRetry() throws Exception { + Response successResponse = successResponse("main/1_7_5.json"); + Response rejectionResponse = rejectionResponse429(); + AtomicInteger callCount = new AtomicInteger(0); + + doAnswer(inv -> { + org.elasticsearch.client.ResponseListener listener = inv.getArgument(1); + if (callCount.getAndIncrement() == 0) { + listener.onFailure(new org.elasticsearch.client.ResponseException(rejectionResponse)); + } else { + listener.onSuccess(successResponse); + } + return null; + }).when(client).performRequestAsync(any(), any()); + + AtomicBoolean success = new AtomicBoolean(false); + AtomicInteger retryCount = new AtomicInteger(0); + + RemoteReindexingUtils.lookupRemoteVersionWithRetries( + logger, + BackoffPolicy.constantBackoff(TimeValue.ZERO, 1), + threadPool, + client, + retryCount::incrementAndGet, + RejectAwareActionListener.wrap( + v -> { + assertEquals(Version.fromString("1.7.5"), v); + success.set(true); + }, + e -> fail("unexpected failure"), + e -> fail("unexpected rejection") + ) + ); + + assertTrue("listener should have received success", success.get()); + assertEquals("countRetry should be invoked once per retry", 1, retryCount.get()); + assertEquals("performRequestAsync should be called twice (initial + 1 retry)", 2, callCount.get()); + } + + /** + * Verifies that lookupRemoteVersionWithRetries propagates failure when retries are exhausted. + */ + public void testLookupRemoteVersionWithRetriesExhaustedPropagatesFailure() throws Exception { + Response rejectionResponse = rejectionResponse429(); + doAnswer(inv -> { + ((org.elasticsearch.client.ResponseListener) inv.getArgument(1)).onFailure( + new org.elasticsearch.client.ResponseException(rejectionResponse) + ); + return null; + }).when(client).performRequestAsync(any(), any()); + + AtomicBoolean failed = new AtomicBoolean(false); + + RemoteReindexingUtils.lookupRemoteVersionWithRetries( + logger, + BackoffPolicy.constantBackoff(TimeValue.ZERO, 1), + threadPool, + client, + () -> {}, + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(RestStatus.TOO_MANY_REQUESTS, ((ElasticsearchStatusException) e).status()); + failed.set(true); + }, + e -> fail("should have propagated as failure after retries exhausted") + ) + ); + + assertTrue("listener should have received failure", failed.get()); + verify(client, times(2)).performRequestAsync(any(), any()); + } + + /** + * Verifies that non-429 errors do not trigger retries. + */ + public void testLookupRemoteVersionWithRetriesNon429DoesNotRetry() throws Exception { + Response badRequestResponse = mock(Response.class); + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR.getStatus()); + when(badRequestResponse.getStatusLine()).thenReturn(statusLine); + when(badRequestResponse.getEntity()).thenReturn(new StringEntity("error", ContentType.TEXT_PLAIN)); + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("GET"); + when(badRequestResponse.getRequestLine()).thenReturn(requestLine); + + mockFailure(new org.elasticsearch.client.ResponseException(badRequestResponse)); + + RemoteReindexingUtils.lookupRemoteVersionWithRetries( + logger, + BackoffPolicy.constantBackoff(TimeValue.ZERO, 5), + threadPool, + client, + () -> fail("countRetry should not be called for non-429"), + RejectAwareActionListener.wrap( + v -> fail(), + e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, ((ElasticsearchStatusException) e).status()); + }, + e -> fail() + ) + ); + + verify(client, times(1)).performRequestAsync(any(), any()); + } + + /** + * Verifies that success on the first attempt does not invoke countRetry. + */ + public void testLookupRemoteVersionWithRetriesSucceedsOnFirstCall() throws Exception { + Response successResponse = successResponse("main/2_3_3.json"); + mockSuccess(successResponse); + + AtomicBoolean success = new AtomicBoolean(false); + + RemoteReindexingUtils.lookupRemoteVersionWithRetries( + logger, + BackoffPolicy.constantBackoff(TimeValue.ZERO, 5), + threadPool, + client, + () -> fail("countRetry should not be called when first attempt succeeds"), + RejectAwareActionListener.wrap( + v -> { + assertEquals(Version.fromString("2.3.3"), v); + success.set(true); + }, + e -> fail(), + e -> fail() + ) + ); + + assertTrue("listener should have received success", success.get()); + verify(client, times(1)).performRequestAsync(any(), any()); + } + + private Response successResponse(String resource) throws Exception { + URL url = Thread.currentThread().getContextClassLoader().getResource("responses/" + resource); + assertNotNull("missing test resource [" + resource + "]", url); + HttpEntity entity = new InputStreamEntity(FileSystemUtils.openFileURLStream(url), ContentType.APPLICATION_JSON); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + return response; + } + + private Response rejectionResponse429() { + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(null); + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(RestStatus.TOO_MANY_REQUESTS.getStatus()); + when(response.getStatusLine()).thenReturn(statusLine); + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("GET"); + when(response.getRequestLine()).thenReturn(requestLine); + return response; + } + private void mockSuccess(Response response) { doAnswer(inv -> { ((org.elasticsearch.client.ResponseListener) inv.getArgument(1)).onSuccess(response); From 8552f606f3f870256a05a584df803b71c134e85f Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 11:50:32 +0000 Subject: [PATCH 19/45] Merge main --- .../org/elasticsearch/reindex/Reindexer.java | 4 +- .../RemoteScrollablePaginatedHitSource.java | 4 +- .../remote/RemoteReindexingUtilsTests.java | 50 +++++++------------ 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 679870b959b71..b88fa603a6d76 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -223,12 +223,12 @@ public void onResponse(Version version) { @Override public void onFailure(Exception e) { - closeRestClientAndRun(restClient, () -> listener.onFailure(e)); + closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)); } @Override public void onRejection(Exception e) { - closeRestClientAndRun(restClient, () -> listener.onFailure(e)); + closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)); } }; RemoteReindexingUtils.lookupRemoteVersionWithRetries( diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java index 2d3ce4cb14579..cf5dd07e111ec 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollablePaginatedHitSource.java @@ -29,9 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; -import java.io.InputStream; import java.util.Optional; -import java.util.function.BiFunction; import java.util.function.Consumer; import static org.elasticsearch.core.Strings.format; @@ -126,7 +124,7 @@ public Optional remoteVersion() { } // Exposed for testing - private void onStartResponse(RejectAwareActionListener searchListener, Response response) { + void onStartResponse(RejectAwareActionListener searchListener, Response response) { if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) { logger.debug("First response looks like a scan response. Jumping right to the second. scroll=[{}]", response.getScrollId()); doStartNextScroll(response.getScrollId(), timeValueMillis(0), searchListener); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index cd677a9d92ab9..2798f7457aa07 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -336,14 +336,10 @@ public void testLookupRemoteVersionWithRetriesSucceedsOnRetry() throws Exception threadPool, client, retryCount::incrementAndGet, - RejectAwareActionListener.wrap( - v -> { - assertEquals(Version.fromString("1.7.5"), v); - success.set(true); - }, - e -> fail("unexpected failure"), - e -> fail("unexpected rejection") - ) + RejectAwareActionListener.wrap(v -> { + assertEquals(Version.fromString("1.7.5"), v); + success.set(true); + }, e -> fail("unexpected failure"), e -> fail("unexpected rejection")) ); assertTrue("listener should have received success", success.get()); @@ -371,15 +367,11 @@ public void testLookupRemoteVersionWithRetriesExhaustedPropagatesFailure() throw threadPool, client, () -> {}, - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof ElasticsearchStatusException); - assertEquals(RestStatus.TOO_MANY_REQUESTS, ((ElasticsearchStatusException) e).status()); - failed.set(true); - }, - e -> fail("should have propagated as failure after retries exhausted") - ) + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(RestStatus.TOO_MANY_REQUESTS, ((ElasticsearchStatusException) e).status()); + failed.set(true); + }, e -> fail("should have propagated as failure after retries exhausted")) ); assertTrue("listener should have received failure", failed.get()); @@ -407,14 +399,10 @@ public void testLookupRemoteVersionWithRetriesNon429DoesNotRetry() throws Except threadPool, client, () -> fail("countRetry should not be called for non-429"), - RejectAwareActionListener.wrap( - v -> fail(), - e -> { - assertTrue(e instanceof ElasticsearchStatusException); - assertEquals(RestStatus.INTERNAL_SERVER_ERROR, ((ElasticsearchStatusException) e).status()); - }, - e -> fail() - ) + RejectAwareActionListener.wrap(v -> fail(), e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, ((ElasticsearchStatusException) e).status()); + }, e -> fail()) ); verify(client, times(1)).performRequestAsync(any(), any()); @@ -435,14 +423,10 @@ public void testLookupRemoteVersionWithRetriesSucceedsOnFirstCall() throws Excep threadPool, client, () -> fail("countRetry should not be called when first attempt succeeds"), - RejectAwareActionListener.wrap( - v -> { - assertEquals(Version.fromString("2.3.3"), v); - success.set(true); - }, - e -> fail(), - e -> fail() - ) + RejectAwareActionListener.wrap(v -> { + assertEquals(Version.fromString("2.3.3"), v); + success.set(true); + }, e -> fail(), e -> fail()) ); assertTrue("listener should have received success", success.get()); From fd5aed5adb191bb510be0c088398d33707a0fbec Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 15:29:22 +0000 Subject: [PATCH 20/45] Open and Close PIT for reindexing As part of the new reindexing resilience work, we're migrating from using scroll search to using point-in-time (PIT). This change opens a PIT (if we're behind a feature flag) before slicing a request, and closes a PIT once the reindexing is complete. It does not pass the PIT ID into the worker actions to use, however; this will be completed in a follow-up PR. Relates: https://github.com/elastic/elasticsearch-team/issues/2088 --- .../org/elasticsearch/reindex/Reindexer.java | 149 ++++++++++++++---- .../reindex/remote/RemoteReindexingUtils.java | 43 +++++ .../reindex/remote/RemoteRequestBuilders.java | 20 +++ .../reindex/remote/RemoteResponseParsers.java | 27 ++++ 4 files changed, 207 insertions(+), 32 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index b88fa603a6d76..554c614fa6218 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -25,7 +25,11 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.internal.Client; @@ -45,6 +49,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -95,6 +100,8 @@ import static org.elasticsearch.common.BackoffPolicy.exponentialBackoff; import static org.elasticsearch.index.VersionType.INTERNAL; import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.closePit; +import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.openPit; public class Reindexer { @@ -169,30 +176,81 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl searchAction.start(); }; - /** - * If this is a request to reindex from remote, then we need to determine the remote version prior to execution - * NB {@link ReindexRequest} forbids remote requests and slices > 1, so we're guaranteed to be running on the only slice - */ - if (REINDEX_PIT_SEARCH_ENABLED && request.getRemoteInfo() != null) { + // Point-in-time searching is not enabled, so default to scroll + if (REINDEX_PIT_SEARCH_ENABLED == false) { + executePaginatedSearch(task, request, listenerWithRelocations, workerAction, null); + } + // Point-in-time searching is enabled, and this is a remote request + else if (request.getRemoteInfo() != null) { lookupRemoteVersionAndExecute(task, request, listenerWithRelocations, workerAction); - } else { - BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listenerWithRelocations, - client, - clusterService.localNode(), - null, - workerAction - ); } + // Point-in-time searching is enabled, and this is a local request + else { + openPitAndExecute(task, request, listenerWithRelocations, workerAction); + } + } + + /** + * Returns the keep-alive duration for PIT. Uses the request's scroll time when set, otherwise defaults to 5 minutes. + */ + private static TimeValue pitKeepAlive(ReindexRequest request) { + return request.getScrollTime() != null ? request.getScrollTime() : TimeValue.timeValueMinutes(5); + } + + // TODO - If we refactor and only use once, then inline + /** + * Runs the sliced action using scroll. + */ + private void executePaginatedSearch( + BulkByScrollTask task, + ReindexRequest request, + ActionListener listener, + Consumer workerAction, + @Nullable Version remoteVersion + ) { + BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( + task, + request, + ReindexAction.INSTANCE, + listener, + client, + clusterService.localNode(), + remoteVersion, + workerAction + ); + } + + /** + * Opens a PIT on the local cluster, runs the sliced action, and closes the PIT when done. + */ + private void openPitAndExecute( + BulkByScrollTask task, + ReindexRequest request, + ActionListener listenerWithRelocations, + Consumer workerAction + ) { + SearchRequest searchRequest = request.getSearchRequest(); + String[] indices = searchRequest.indices(); + OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices).indicesOptions(searchRequest.indicesOptions()) + .keepAlive(pitKeepAlive(request)); + client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithRelocations.delegateFailureAndWrap((l, pitResponse) -> { + BytesReference pitId = pitResponse.getPointInTimeId(); + ActionListener listenerWithClosePit = ActionListener.runAfter( + l, + () -> client.execute( + TransportClosePointInTimeAction.TYPE, + new ClosePointInTimeRequest(pitId), + ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) + ) + ); + executePaginatedSearch(task, request, listenerWithClosePit, workerAction, null); + })); } /** - * Looks up the remote cluster version when reindexing from a remote source, then runs the sliced action with that version. - * The RestClient used for the lookup is closed after the callback; closing must happen on a thread other than the - * RestClient's own thread pool to avoid shutdown failures. + * Looks up the remote cluster version when reindexing from a remote source. If the remote supports PIT (7.10.0+), + * opens PIT, runs the sliced action, and closes PIT when done. Otherwise uses scroll. + * The RestClient used for lookup (and PIT open/close when applicable) is closed after completion. */ private void lookupRemoteVersionAndExecute( BulkByScrollTask task, @@ -206,19 +264,15 @@ private void lookupRemoteVersionAndExecute( RejectAwareActionListener rejectAwareListener = new RejectAwareActionListener<>() { @Override public void onResponse(Version version) { - closeRestClientAndRun( - restClient, - () -> BulkByPaginatedSearchParallelizationHelper.executeSlicedAction( - task, - request, - ReindexAction.INSTANCE, - listenerWithRelocations, - client, - clusterService.localNode(), - version, - workerAction - ) - ); + boolean canUsePit = version.onOrAfter(Version.V_7_10_0); + if (canUsePit) { + openRemotePitAndExecute(task, request, listenerWithRelocations, workerAction, restClient, version); + } else { + closeRestClientAndRun( + restClient, + () -> executePaginatedSearch(task, request, listenerWithRelocations, workerAction, version) + ); + } } @Override @@ -241,6 +295,37 @@ public void onRejection(Exception e) { ); } + /** + * Opens a PIT on the remote cluster, runs the sliced action, and closes the PIT when done. + * The RestClient is closed after the PIT is closed. + */ + private void openRemotePitAndExecute( + BulkByScrollTask task, + ReindexRequest request, + ActionListener listenerWithRelocations, + Consumer workerAction, + RestClient restClient, + Version remoteVersion + ) { + String[] indices = request.getSearchRequest().indices(); + openPit(indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { + ActionListener listenerWithClosePit = ActionListener.runAfter( + listenerWithRelocations, + () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { + logger.warn("Failed to close remote PIT", e); + closeRestClientAndRun(restClient, () -> {}); + }, e -> { + logger.warn("Failed to close remote PIT (rejected)", e); + closeRestClientAndRun(restClient, () -> {}); + }), threadPool, restClient) + ); + executePaginatedSearch(task, request, listenerWithClosePit, workerAction, remoteVersion); + }, + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) + ), threadPool, restClient); + } + /** * Closes the RestClient on the generic thread pool (to avoid closing from the client's own thread), then runs the given action. */ diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 36abf08337f96..9061845a321dc 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -22,9 +22,11 @@ import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.BackoffPolicy; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RetryListener; import org.elasticsearch.rest.RestStatus; @@ -40,6 +42,7 @@ import java.util.function.Supplier; import static org.elasticsearch.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER; +import static org.elasticsearch.reindex.remote.RemoteResponseParsers.OPEN_PIT_PARSER; /** * Utility methods for reindexing from remote Elasticsearch clusters. @@ -58,6 +61,46 @@ public static void lookupRemoteVersion(RejectAwareActionListener listen execute(new Request("GET", "/"), MAIN_ACTION_PARSER, listener, threadPool, client); } + /** + * Opens a point-in-time on the remote cluster. Requires remote version 7.10.0 or later. + * + * @param indices indices to open PIT on + * @param keepAlive PIT keep alive duration + * @param listener receives the PIT id on success, or failure/rejection on error + * @param threadPool thread pool for preserving thread context + * @param client REST client for the remote cluster + */ + public static void openPit( + String[] indices, + TimeValue keepAlive, + RejectAwareActionListener listener, + ThreadPool threadPool, + RestClient client + ) { + execute(RemoteRequestBuilders.openPit(indices, keepAlive), OPEN_PIT_PARSER, listener, threadPool, client); + } + + /** + * Closes a point-in-time on the remote cluster. + * + * @param pitId the PIT id to close + * @param listener receives on success, or failure on error + * @param threadPool thread pool for preserving thread context + * @param client REST client for the remote cluster + */ + public static void closePit(BytesReference pitId, RejectAwareActionListener listener, ThreadPool threadPool, RestClient client) { + execute(RemoteRequestBuilders.closePit(pitId), (p, xContentType) -> { + try { + if (p.nextToken() != null) { + p.skipChildren(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + }, RejectAwareActionListener.withResponseHandler(listener, v -> listener.onResponse(null)), threadPool, client); + } + /** * Looks up the remote cluster version with retries on rejection (e.g. 429 Too Many Requests). * Matches the retry behavior used by {@link RemoteScrollablePaginatedHitSource} when it looks up the version. diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java index 1c5a877331b6b..ede9b6607768c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java @@ -178,6 +178,26 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query, return request; } + static Request openPit(String[] indices, TimeValue keepAlive) { + StringBuilder path = new StringBuilder("/"); + addIndices(path, indices); + path.append("_pit"); + Request request = new Request("POST", path.toString()); + request.addParameter("keep_alive", keepAlive.getStringRep()); + return request; + } + + static Request closePit(BytesReference pitId) { + Request request = new Request("DELETE", "/_pit"); + try (XContentBuilder entity = JsonXContent.contentBuilder()) { + entity.startObject().field("id", java.util.Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(pitId))).endObject(); + request.setJsonEntity(Strings.toString(entity)); + } catch (IOException e) { + throw new ElasticsearchException("failed to build close pit entity", e); + } + return request; + } + private static void addIndices(StringBuilder path, String[] indices) { if (indices == null || indices.length == 0) { return; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java index 818aa7670e0cb..7468f9599231c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Tuple; @@ -30,6 +31,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Base64; import java.util.List; import java.util.function.BiFunction; @@ -269,6 +271,31 @@ public void setCausedBy(Throwable causedBy) { } } + /** + * Parser for the open point-in-time response. Returns the PIT id as {@link BytesReference}. + */ + public static final BiFunction OPEN_PIT_PARSER = (p, xContentType) -> { + try { + String id = null; + if (p.nextToken() != XContentParser.Token.START_OBJECT) { + throw new IllegalArgumentException("open point-in-time response must be an object"); + } + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + if (p.currentToken() == XContentParser.Token.FIELD_NAME && "id".equals(p.currentName())) { + p.nextToken(); + id = p.text(); + break; + } + } + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("open point-in-time response must contain [id] field"); + } + return new BytesArray(Base64.getUrlDecoder().decode(id)); + } catch (IOException e) { + throw new RuntimeException("Failed to parse open point-in-time response", e); + } + }; + /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ From 710daf6ede4f2fd2a169742a1e7f02721ff8ed38 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 15:47:35 +0000 Subject: [PATCH 21/45] Fix unit tests --- .../org/elasticsearch/reindex/Reindexer.java | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 554c614fa6218..27f4ffb1c6432 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -157,7 +157,36 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl // for update-by-query and delete-by-query final ActionListener listenerWithRelocations = listenerWithRelocations(task, request, listener); - Consumer workerAction = remoteVersion -> { + final boolean isRemote = request.getRemoteInfo() != null; + Consumer workerAction = createWorkerAction(task, request, bulkClient, listenerWithRelocations, startTime, isRemote); + + // Point-in-time searching is not enabled, so default to scroll + if (REINDEX_PIT_SEARCH_ENABLED == false) { + executePaginatedSearch(task, request, listenerWithRelocations, workerAction, null); + } + // Point-in-time searching is enabled, and this is a remote request + else if (isRemote) { + lookupRemoteVersionAndExecute(task, request, bulkClient, listenerWithRelocations, workerAction, startTime); + } + // Point-in-time searching is enabled, and this is a local request + else { + openPitAndExecute(task, request, bulkClient, listenerWithRelocations, startTime); + } + } + + /** + * Creates the worker action that runs the reindex. The listener is invoked on success or failure; for PIT paths, + * the listener should include runAfter logic to close the PIT. + */ + private Consumer createWorkerAction( + BulkByScrollTask task, + ReindexRequest request, + Client bulkClient, + ActionListener listener, + long startTime, + boolean isRemote + ) { + return remoteVersion -> { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); ParentTaskAssigningClient assigningBulkClient = new ParentTaskAssigningClient(bulkClient, clusterService.localNode(), task); AsyncIndexBySearchAction searchAction = new AsyncIndexBySearchAction( @@ -170,24 +199,11 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl projectResolver.getProjectState(clusterService.state()), reindexSslConfig, request, - workerListenerWithRelocationAndMetrics(listenerWithRelocations, startTime, request.getRemoteInfo() != null), + workerListenerWithRelocationAndMetrics(listener, startTime, isRemote), remoteVersion ); searchAction.start(); }; - - // Point-in-time searching is not enabled, so default to scroll - if (REINDEX_PIT_SEARCH_ENABLED == false) { - executePaginatedSearch(task, request, listenerWithRelocations, workerAction, null); - } - // Point-in-time searching is enabled, and this is a remote request - else if (request.getRemoteInfo() != null) { - lookupRemoteVersionAndExecute(task, request, listenerWithRelocations, workerAction); - } - // Point-in-time searching is enabled, and this is a local request - else { - openPitAndExecute(task, request, listenerWithRelocations, workerAction); - } } /** @@ -197,7 +213,6 @@ private static TimeValue pitKeepAlive(ReindexRequest request) { return request.getScrollTime() != null ? request.getScrollTime() : TimeValue.timeValueMinutes(5); } - // TODO - If we refactor and only use once, then inline /** * Runs the sliced action using scroll. */ @@ -226,8 +241,9 @@ private void executePaginatedSearch( private void openPitAndExecute( BulkByScrollTask task, ReindexRequest request, + Client bulkClient, ActionListener listenerWithRelocations, - Consumer workerAction + long startTime ) { SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); @@ -243,7 +259,8 @@ private void openPitAndExecute( ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) ) ); - executePaginatedSearch(task, request, listenerWithClosePit, workerAction, null); + Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit, startTime, false); + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); })); } @@ -255,8 +272,10 @@ private void openPitAndExecute( private void lookupRemoteVersionAndExecute( BulkByScrollTask task, ReindexRequest request, + Client bulkClient, ActionListener listenerWithRelocations, - Consumer workerAction + Consumer workerAction, + long startTime ) { RemoteInfo remoteInfo = request.getRemoteInfo(); assert reindexSslConfig != null : "Reindex ssl config must be set"; @@ -266,7 +285,7 @@ private void lookupRemoteVersionAndExecute( public void onResponse(Version version) { boolean canUsePit = version.onOrAfter(Version.V_7_10_0); if (canUsePit) { - openRemotePitAndExecute(task, request, listenerWithRelocations, workerAction, restClient, version); + openRemotePitAndExecute(task, request, bulkClient, listenerWithRelocations, restClient, version, startTime); } else { closeRestClientAndRun( restClient, @@ -302,10 +321,11 @@ public void onRejection(Exception e) { private void openRemotePitAndExecute( BulkByScrollTask task, ReindexRequest request, + Client bulkClient, ActionListener listenerWithRelocations, - Consumer workerAction, RestClient restClient, - Version remoteVersion + Version remoteVersion, + long startTime ) { String[] indices = request.getSearchRequest().indices(); openPit(indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { @@ -319,7 +339,8 @@ private void openRemotePitAndExecute( closeRestClientAndRun(restClient, () -> {}); }), threadPool, restClient) ); - executePaginatedSearch(task, request, listenerWithClosePit, workerAction, remoteVersion); + Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit, startTime, true); + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); }, e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) From 6790f2f9a04695b3fe79dc0fa3a02f6639518a42 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 16:01:51 +0000 Subject: [PATCH 22/45] Wrap listener in metrics --- .../org/elasticsearch/reindex/Reindexer.java | 60 ++++++++++++++++--- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 27f4ffb1c6432..082d965bae532 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -158,7 +158,15 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl final ActionListener listenerWithRelocations = listenerWithRelocations(task, request, listener); final boolean isRemote = request.getRemoteInfo() != null; - Consumer workerAction = createWorkerAction(task, request, bulkClient, listenerWithRelocations, startTime, isRemote); + Consumer workerAction = createWorkerAction( + task, + request, + bulkClient, + listenerWithRelocations, + startTime, + isRemote, + false + ); // Point-in-time searching is not enabled, so default to scroll if (REINDEX_PIT_SEARCH_ENABLED == false) { @@ -184,8 +192,14 @@ private Consumer createWorkerAction( Client bulkClient, ActionListener listener, long startTime, - boolean isRemote + boolean isRemote, + boolean listenerAlreadyHasMetrics ) { + // PIT paths wrap the listener with metrics before executing PIT specific REST calls. + // When scroll is then used, we avoid double-recording by skipping the wrapper here. + ActionListener workerListener = listenerAlreadyHasMetrics + ? listener + : workerListenerWithRelocationAndMetrics(listener, startTime, isRemote); return remoteVersion -> { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); ParentTaskAssigningClient assigningBulkClient = new ParentTaskAssigningClient(bulkClient, clusterService.localNode(), task); @@ -199,7 +213,7 @@ private Consumer createWorkerAction( projectResolver.getProjectState(clusterService.state()), reindexSslConfig, request, - workerListenerWithRelocationAndMetrics(listener, startTime, isRemote), + workerListener, remoteVersion ); searchAction.start(); @@ -245,11 +259,17 @@ private void openPitAndExecute( ActionListener listenerWithRelocations, long startTime ) { + // Wrap with metrics so failures before the worker runs (PIT open) are recorded + final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( + listenerWithRelocations, + startTime, + false + ); SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices).indicesOptions(searchRequest.indicesOptions()) .keepAlive(pitKeepAlive(request)); - client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithRelocations.delegateFailureAndWrap((l, pitResponse) -> { + client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { BytesReference pitId = pitResponse.getPointInTimeId(); ActionListener listenerWithClosePit = ActionListener.runAfter( l, @@ -259,7 +279,15 @@ private void openPitAndExecute( ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) ) ); - Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit, startTime, false); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + false, + true + ); executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); })); } @@ -277,6 +305,12 @@ private void lookupRemoteVersionAndExecute( Consumer workerAction, long startTime ) { + // Wrap with metrics so failures before the worker runs (version lookup, PIT open) are recorded + final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( + listenerWithRelocations, + startTime, + true + ); RemoteInfo remoteInfo = request.getRemoteInfo(); assert reindexSslConfig != null : "Reindex ssl config must be set"; RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); @@ -285,7 +319,7 @@ private void lookupRemoteVersionAndExecute( public void onResponse(Version version) { boolean canUsePit = version.onOrAfter(Version.V_7_10_0); if (canUsePit) { - openRemotePitAndExecute(task, request, bulkClient, listenerWithRelocations, restClient, version, startTime); + openRemotePitAndExecute(task, request, bulkClient, listenerWithMetrics, restClient, version, startTime); } else { closeRestClientAndRun( restClient, @@ -296,12 +330,12 @@ public void onResponse(Version version) { @Override public void onFailure(Exception e) { - closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)); + closeRestClientAndRun(restClient, () -> listenerWithMetrics.onFailure(e)); } @Override public void onRejection(Exception e) { - closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)); + closeRestClientAndRun(restClient, () -> listenerWithMetrics.onFailure(e)); } }; RemoteReindexingUtils.lookupRemoteVersionWithRetries( @@ -339,7 +373,15 @@ private void openRemotePitAndExecute( closeRestClientAndRun(restClient, () -> {}); }), threadPool, restClient) ); - Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit, startTime, true); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + true, + true + ); executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); }, e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), From 33783226a9b1c4f0715d9ad0262a6b7295105e2d Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 25 Feb 2026 16:33:42 +0000 Subject: [PATCH 23/45] Add unit tests for RemoteReindexingUtils, RemoteRequestBuilders and RemoteResponseParsers --- .../org/elasticsearch/reindex/Reindexer.java | 10 +- .../remote/RemoteReindexingUtilsTests.java | 237 ++++++++++++++++++ .../remote/RemoteRequestBuildersTests.java | 130 ++++++++++ .../remote/RemoteResponseParsersTests.java | 85 +++++++ 4 files changed, 453 insertions(+), 9 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 082d965bae532..d70c06d5f3117 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -158,15 +158,7 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl final ActionListener listenerWithRelocations = listenerWithRelocations(task, request, listener); final boolean isRemote = request.getRemoteInfo() != null; - Consumer workerAction = createWorkerAction( - task, - request, - bulkClient, - listenerWithRelocations, - startTime, - isRemote, - false - ); + Consumer workerAction = createWorkerAction(task, request, bulkClient, listenerWithRelocations, startTime, isRemote, false); // Point-in-time searching is not enabled, so default to scroll if (REINDEX_PIT_SEARCH_ENABLED == false) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 2798f7457aa07..8459d3bbdbd79 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -20,6 +20,8 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.client.Response; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.io.FileSystemUtils; @@ -43,6 +45,7 @@ import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.wrapExceptionToPreserveStatus; import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertArrayEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -433,6 +436,240 @@ public void testLookupRemoteVersionWithRetriesSucceedsOnFirstCall() throws Excep verify(client, times(1)).performRequestAsync(any(), any()); } + /** + * Verifies that openPit parses a valid open PIT response and invokes onResponse with the decoded PIT id. + */ + public void testOpenPitSuccess() { + byte[] pitIdBytes = randomByteArrayOfLength(between(1, 64)); + String base64Id = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + String json = "{\"id\":\"" + base64Id + "\"}"; + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); + mockSuccess(response); + + AtomicBoolean success = new AtomicBoolean(false); + BytesReference[] capturedPitId = new BytesReference[1]; + RemoteReindexingUtils.openPit( + new String[] { randomAlphaOfLength(between(1, 10)) }, + TimeValue.timeValueMillis(between(1, 60000)), + RejectAwareActionListener.wrap( + pitId -> { + capturedPitId[0] = pitId; + success.set(true); + }, + e -> fail("unexpected failure"), + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("listener should have received success", success.get()); + assertArrayEquals(pitIdBytes, BytesReference.toBytes(capturedPitId[0])); + } + + /** + * Verifies that openPit invokes onRejection when the remote returns HTTP 429. + */ + public void testOpenPitTooManyRequestsTriggersRejection() throws Exception { + mockFailure(new org.elasticsearch.client.ResponseException(rejectionResponse429())); + + AtomicBoolean rejected = new AtomicBoolean(false); + RemoteReindexingUtils.openPit( + new String[] { randomAlphaOfLength(between(1, 10)) }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> fail("unexpected failure"), e -> rejected.set(true)), + threadPool, + client + ); + assertTrue("onRejection should have been called", rejected.get()); + } + + /** + * Verifies that openPit invokes onFailure when the remote returns a non-429 HTTP error. + */ + public void testOpenPitHttpErrorTriggersFailure() throws Exception { + int statusCode = randomFrom(RestStatus.BAD_REQUEST, RestStatus.NOT_FOUND, RestStatus.INTERNAL_SERVER_ERROR).getStatus(); + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + Response response = mock(Response.class); + when(response.getStatusLine()).thenReturn(statusLine); + when(response.getEntity()).thenReturn(new StringEntity(randomAlphaOfLength(between(1, 20)), ContentType.TEXT_PLAIN)); + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("POST"); + when(response.getRequestLine()).thenReturn(requestLine); + mockFailure(new org.elasticsearch.client.ResponseException(response)); + + AtomicBoolean failed = new AtomicBoolean(false); + RemoteReindexingUtils.openPit( + new String[] { randomAlphaOfLength(between(1, 10)) }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); + failed.set(true); + }, + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("onFailure should have been called", failed.get()); + } + + /** + * Verifies that openPit invokes onFailure when the response body is invalid JSON. + */ + public void testOpenPitInvalidJsonTriggersFailure() { + String invalidJson = randomAlphaOfLength(between(5, 20)) + "!!!"; + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(new StringEntity(invalidJson, ContentType.APPLICATION_JSON)); + mockSuccess(response); + + AtomicBoolean failed = new AtomicBoolean(false); + RemoteReindexingUtils.openPit( + new String[] { randomAlphaOfLength(between(1, 10)) }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof ElasticsearchException); + assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); + failed.set(true); + }, + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("onFailure should have been called", failed.get()); + } + + /** + * Verifies that openPit invokes onFailure when the response is valid JSON but missing the required id field. + */ + public void testOpenPitMissingIdFieldTriggersFailure() { + String json = "{\"other\":\"" + randomAlphaOfLength(between(1, 10)) + "\"}"; + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); + mockSuccess(response); + + AtomicBoolean failed = new AtomicBoolean(false); + RemoteReindexingUtils.openPit( + new String[] { randomAlphaOfLength(between(1, 10)) }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof IllegalArgumentException); + assertThat(e.getMessage(), containsString("open point-in-time response must contain [id] field")); + failed.set(true); + }, + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("onFailure should have been called", failed.get()); + } + + /** + * Verifies that closePit invokes onResponse when the remote returns a successful close PIT response. + */ + public void testClosePitSuccess() { + String json = "{\"succeeded\":" + randomBoolean() + "}"; + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); + mockSuccess(response); + + AtomicBoolean success = new AtomicBoolean(false); + BytesReference pitId = new BytesArray(randomByteArrayOfLength(between(1, 32))); + RemoteReindexingUtils.closePit( + pitId, + RejectAwareActionListener.wrap(v -> success.set(true), e -> fail("unexpected failure"), e -> fail("unexpected rejection")), + threadPool, + client + ); + assertTrue("listener should have received success", success.get()); + } + + /** + * Verifies that closePit invokes onRejection when the remote returns HTTP 429. + */ + public void testClosePitTooManyRequestsTriggersRejection() throws Exception { + mockFailure(new org.elasticsearch.client.ResponseException(rejectionResponse429())); + + AtomicBoolean rejected = new AtomicBoolean(false); + RemoteReindexingUtils.closePit( + new BytesArray(randomByteArrayOfLength(between(1, 32))), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> fail("unexpected failure"), e -> rejected.set(true)), + threadPool, + client + ); + assertTrue("onRejection should have been called", rejected.get()); + } + + /** + * Verifies that closePit invokes onFailure when the remote returns a non-429 HTTP error. + */ + public void testClosePitHttpErrorTriggersFailure() throws Exception { + int statusCode = randomFrom(RestStatus.BAD_REQUEST, RestStatus.NOT_FOUND, RestStatus.INTERNAL_SERVER_ERROR).getStatus(); + org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + Response response = mock(Response.class); + when(response.getStatusLine()).thenReturn(statusLine); + when(response.getEntity()).thenReturn(new StringEntity(randomAlphaOfLength(between(1, 20)), ContentType.TEXT_PLAIN)); + RequestLine requestLine = mock(RequestLine.class); + when(requestLine.getMethod()).thenReturn("DELETE"); + when(response.getRequestLine()).thenReturn(requestLine); + mockFailure(new org.elasticsearch.client.ResponseException(response)); + + AtomicBoolean failed = new AtomicBoolean(false); + RemoteReindexingUtils.closePit( + new BytesArray(randomByteArrayOfLength(between(1, 32))), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); + failed.set(true); + }, + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("onFailure should have been called", failed.get()); + } + + /** + * Verifies that closePit invokes onFailure when the response body is invalid JSON. + */ + public void testClosePitInvalidJsonTriggersFailure() { + String invalidJson = randomAlphaOfLength(between(5, 20)) + "!!!"; + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(new StringEntity(invalidJson, ContentType.APPLICATION_JSON)); + mockSuccess(response); + + AtomicBoolean failed = new AtomicBoolean(false); + RemoteReindexingUtils.closePit( + new BytesArray(randomByteArrayOfLength(between(1, 32))), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + e -> { + assertTrue(e instanceof ElasticsearchException); + assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); + failed.set(true); + }, + e -> fail("unexpected rejection") + ), + threadPool, + client + ); + assertTrue("onFailure should have been called", failed.get()); + } + private Response successResponse(String resource) throws Exception { URL url = Thread.currentThread().getContextClassLoader().getResource("responses/" + resource); assertNotNull("missing test resource [" + resource + "]", url); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java index de2e0b6379536..505295f91ce08 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java @@ -30,7 +30,9 @@ import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.reindex.remote.RemoteRequestBuilders.clearScroll; +import static org.elasticsearch.reindex.remote.RemoteRequestBuilders.closePit; import static org.elasticsearch.reindex.remote.RemoteRequestBuilders.initialSearch; +import static org.elasticsearch.reindex.remote.RemoteRequestBuilders.openPit; import static org.elasticsearch.reindex.remote.RemoteRequestBuilders.scroll; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -40,6 +42,8 @@ import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; /** * Tests for {@link RemoteRequestBuilders} which builds requests for remote version of @@ -315,4 +319,130 @@ public void testClearScroll() throws IOException { assertEquals(scroll, Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8))); assertThat(request.getParameters().keySet(), empty()); } + + /** + * Verifies that openPit builds a POST request to the correct path for a single index. + */ + public void testOpenPitSingleIndex() { + String index = randomAlphaOfLength(between(1, 20)); + TimeValue keepAlive = randomPositiveTimeValue(); + Request request = openPit(new String[] { index }, keepAlive); + assertEquals("POST", request.getMethod()); + assertEquals("/" + index + "/_pit", request.getEndpoint()); + assertThat(request.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + } + + /** + * Verifies that openPit builds a POST request to the correct path for multiple indices. + */ + public void testOpenPitMultipleIndices() { + int numIndices = between(2, 10); + String[] indices = new String[numIndices]; + StringBuilder expectedPath = new StringBuilder("/"); + for (int i = 0; i < numIndices; i++) { + indices[i] = randomAlphaOfLength(between(1, 10)); + if (i > 0) { + expectedPath.append(","); + } + expectedPath.append(indices[i]); + } + expectedPath.append("/_pit"); + TimeValue keepAlive = randomPositiveTimeValue(); + Request request = openPit(indices, keepAlive); + assertEquals("POST", request.getMethod()); + assertEquals(expectedPath.toString(), request.getEndpoint()); + assertThat(request.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + } + + /** + * Verifies that openPit uses /_pit when indices are null or empty, matching addIndices behavior. + */ + public void testOpenPitNullOrEmptyIndices() { + TimeValue keepAlive = randomPositiveTimeValue(); + Request nullRequest = openPit(null, keepAlive); + assertEquals("POST", nullRequest.getMethod()); + assertEquals("/_pit", nullRequest.getEndpoint()); + assertThat(nullRequest.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + + Request emptyRequest = openPit(new String[] {}, keepAlive); + assertEquals("POST", emptyRequest.getMethod()); + assertEquals("/_pit", emptyRequest.getEndpoint()); + assertThat(emptyRequest.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + } + + /** + * Verifies that openPit URL-encodes index names containing special characters (comma, slash). + */ + public void testOpenPitEncodesSpecialCharactersInIndices() { + String prefix1 = randomAlphaOfLength(between(1, 5)); + String prefix2 = randomAlphaOfLength(between(1, 5)); + Request request = openPit(new String[] { prefix1 + ",", prefix2 + "/" }, randomPositiveTimeValue()); + assertEquals("POST", request.getMethod()); + assertEquals("/" + prefix1 + "%2C," + prefix2 + "%2F/_pit", request.getEndpoint()); + } + + /** + * Verifies that openPit passes through various TimeValue formats for keep_alive. + */ + public void testOpenPitKeepAliveParameter() { + String index = randomAlphaOfLength(between(1, 10)); + long millis = between(1, 100000); + assertThat( + openPit(new String[] { index }, timeValueMillis(millis)).getParameters(), + hasEntry("keep_alive", TimeValue.timeValueMillis(millis).getStringRep()) + ); + int minutes = between(1, 60); + assertThat( + openPit(new String[] { index }, TimeValue.timeValueMinutes(minutes)).getParameters(), + hasEntry("keep_alive", TimeValue.timeValueMinutes(minutes).getStringRep()) + ); + int hours = between(1, 24); + assertThat( + openPit(new String[] { index }, TimeValue.timeValueHours(hours)).getParameters(), + hasEntry("keep_alive", TimeValue.timeValueHours(hours).getStringRep()) + ); + } + + /** + * Verifies that closePit builds a DELETE request to /_pit with a JSON body containing the base64-encoded PIT id. + */ + public void testClosePitRequestStructure() throws IOException { + byte[] pitIdBytes = randomByteArrayOfLength(between(1, 64)); + BytesReference pitId = new BytesArray(pitIdBytes); + Request request = closePit(pitId); + assertEquals("DELETE", request.getMethod()); + assertEquals("/_pit", request.getEndpoint()); + assertThat(request.getEntity(), not(nullValue())); + assertEquals(ContentType.APPLICATION_JSON.toString(), request.getEntity().getContentType().getValue()); + String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); + String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + assertThat(body, containsString("\"id\":\"" + expectedId + "\"")); + } + + /** + * Verifies that closePit correctly encodes the PIT id for binary-like content. + */ + public void testClosePitEncodesBinaryPitId() throws IOException { + byte[] pitIdBytes = randomByteArrayOfLength(between(1, 32)); + BytesReference pitId = new BytesArray(pitIdBytes); + Request request = closePit(pitId); + String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); + String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + assertThat(body, containsString("\"id\":\"" + expectedId + "\"")); + } + + /** + * Verifies that closePit produces valid JSON with an id field containing the base64-encoded PIT id. + */ + public void testClosePitProducesValidJson() throws IOException { + String pitIdStr = randomAlphaOfLength(between(1, 50)); + BytesReference pitId = new BytesArray(pitIdStr.getBytes(StandardCharsets.UTF_8)); + Request request = closePit(pitId); + String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); + String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdStr.getBytes(StandardCharsets.UTF_8)); + assertThat(body, containsString("\"id\"")); + assertThat(body, containsString(expectedId)); + assertThat(body.trim(), startsWith("{")); + assertThat(body.trim(), endsWith("}")); + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java index 60b5a70e18225..1b54c7c6638c5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java @@ -10,17 +10,22 @@ package org.elasticsearch.reindex.remote; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.reindex.PaginatedHitSource; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import org.hamcrest.Matchers; import java.io.IOException; +import java.util.Base64; +import static org.elasticsearch.reindex.remote.RemoteResponseParsers.OPEN_PIT_PARSER; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.junit.Assert.assertArrayEquals; public class RemoteResponseParsersTests extends ESTestCase { @@ -38,4 +43,84 @@ public void testFailureWithoutIndex() throws IOException { assertThat(parsed.getReason(), Matchers.instanceOf(EsRejectedExecutionException.class)); } } + + /** + * Verifies that OPEN_PIT_PARSER extracts and base64-url-decodes the id field from a valid open PIT response, + * regardless of field order. + */ + public void testOpenPitParserValidResponse() throws IOException { + byte[] pitIdBytes = randomByteArrayOfLength(between(1, 64)); + String base64Id = Base64.getUrlEncoder().encodeToString(pitIdBytes); + int fieldsBefore = between(0, 3); + int fieldsAfter = between(0, 3); + + XContentBuilder builder = jsonBuilder().startObject(); + // Randomly generates some fields to come before the ID + for (int i = 0; i < fieldsBefore; i++) { + builder.field("before_" + i + "_" + randomAlphaOfLength(between(1, 5)), randomAlphaOfLength(between(1, 10))); + } + builder.field("id", base64Id); + // Randomly generates some fields to come after the ID + for (int i = 0; i < fieldsAfter; i++) { + builder.field("after_" + i + "_" + randomAlphaOfLength(between(1, 5)), randomAlphaOfLength(between(1, 10))); + } + builder.endObject(); + try (XContentParser parser = createParser(builder)) { + BytesReference result = OPEN_PIT_PARSER.apply(parser, XContentType.JSON); + assertArrayEquals(pitIdBytes, BytesReference.toBytes(result)); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the response is an empty object with no id field. + */ + public void testOpenPitParserMissingId() throws IOException { + XContentBuilder builder = jsonBuilder().startObject().endObject(); + try (XContentParser parser = createParser(builder)) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) + ); + assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must contain [id] field")); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the id field is present but empty. + */ + public void testOpenPitParserEmptyId() throws IOException { + XContentBuilder builder = jsonBuilder().startObject().field("id", "").endObject(); + try (XContentParser parser = createParser(builder)) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) + ); + assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must contain [id] field")); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the response is not a JSON object (e.g. an array). + */ + public void testOpenPitParserNotAnObject() throws IOException { + XContentBuilder builder = jsonBuilder().startArray().value("a").endArray(); + try (XContentParser parser = createParser(builder)) { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) + ); + assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must be an object")); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the id field contains invalid base64-url data. + */ + public void testOpenPitParserInvalidBase64() throws IOException { + String invalidBase64 = randomAlphaOfLength(between(1, 20)) + "!!!"; + XContentBuilder builder = jsonBuilder().startObject().field("id", invalidBase64).endObject(); + try (XContentParser parser = createParser(builder)) { + expectThrows(IllegalArgumentException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + } + } } From a03ef5379120f0b73845b5881fd122b5479d3137 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 26 Feb 2026 12:06:00 +0000 Subject: [PATCH 24/45] Add ReindexerTests --- .../elasticsearch/reindex/ReindexerTests.java | 549 ++++++++++++++++++ 1 file changed, 549 insertions(+) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 25e599a9772b0..57cca377566cb 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -10,38 +10,83 @@ package org.elasticsearch.reindex; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; +import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.PaginatedHitSource; import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.ResumeBulkByScrollRequest; import org.elasticsearch.index.reindex.ResumeBulkByScrollResponse; import org.elasticsearch.index.reindex.ResumeInfo; import org.elasticsearch.index.reindex.ResumeReindexAction; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.TaskId; +import org.apache.logging.log4j.Level; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.watcher.ResourceWatcherService; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import static java.util.Collections.emptyMap; +import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE; +import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; import static org.elasticsearch.core.TimeValue.timeValueMillis; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -257,8 +302,512 @@ public void testWorkerListenerRecordsMetricsForNormalResponse() { verifyNoMoreInteractions(metrics, outer); } + /** + * When the remote version lookup fails in lookupRemoteVersionAndExecute + * (e.g. server returns 500), the failure propagates to the listener. + * Uses MockHttpServer instead of a non-connectable host to avoid unreliable connection timeouts. + */ + @SuppressForbidden(reason = "use http server for testing") + public void testRemoteReindexingRequestFailsWhenVersionLookupFails() throws Exception { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(INTERNAL_SERVER_ERROR.getStatus(), -1); + exchange.close(); + }); + server.start(); + try { + runRemotePitTestWithMockServer(server, request -> request.setMaxRetries(0), initFuture -> { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, initFuture::actionGet); + assertThat(e.status(), equalTo(INTERNAL_SERVER_ERROR)); + }); + } finally { + server.stop(0); + } + } + + /** + * When the remote version lookup is rejected (429), the failure propagates to the listener + * after retries are exhausted. + */ + @SuppressForbidden(reason = "use http server for testing") + public void testRemoteReindexingRequestFailsWhenVersionLookupRejected() throws Exception { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(TOO_MANY_REQUESTS.getStatus(), -1); + exchange.close(); + }); + server.start(); + try { + runRemotePitTestWithMockServer(server, request -> request.setMaxRetries(0), initFuture -> { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, initFuture::actionGet); + assertThat(e.status(), equalTo(TOO_MANY_REQUESTS)); + }); + } finally { + server.stop(0); + } + } + + /** + * When opening the remote PIT fails in openRemotePitAndExecute, the failure propagates to the listener. + */ + @SuppressForbidden(reason = "use http server for testing") + public void testRemoteReindexingRequestFailsToOpenPit() throws Exception { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + AtomicInteger requestCount = new AtomicInteger(0); + HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + int count = requestCount.getAndIncrement(); + if (count == 0) { + respondJson(exchange, 200, REMOTE_PIT_TEST_VERSION_JSON); + } else { + exchange.sendResponseHeaders(500, -1); + } + exchange.close(); + }); + server.start(); + try { + runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, initFuture::actionGet); + assertThat(e.status(), equalTo(INTERNAL_SERVER_ERROR)); + }); + } finally { + server.stop(0); + } + } + + /** + * When closing the remote PIT fails in openRemotePitAndExecute, the failure is logged + * but the main listener still receives success. + */ + @SuppressForbidden(reason = "use http server for testing") + public void testRemoteReindexingRequestFailsToClosePit() throws Exception { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + HttpServer server = createRemotePitMockServer( + (path, method) -> path.contains("_pit") && "DELETE".equals(method), + exchange -> { + try { + exchange.sendResponseHeaders(500, -1); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + server.start(); + try { + MockLog.awaitLogger( + () -> { + try { + runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { + BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + Reindexer.class, + new MockLog.SeenEventExpectation( + "Failed to close remote PIT should be logged", + Reindexer.class.getCanonicalName(), + Level.WARN, + "Failed to close remote PIT" + ) + ); + } finally { + server.stop(0); + } + } + + /** + * When closing the remote PIT is rejected (429) in openRemotePitAndExecute, + * the rejection is logged but the main listener still receives success. + */ + @SuppressForbidden(reason = "use http server for testing") + public void testRemoteReindexingRequestFailsWhenClosePitIsRejected() throws Exception { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + HttpServer server = createRemotePitMockServer( + (path, method) -> path.contains("_pit") && "DELETE".equals(method), + exchange -> { + try { + exchange.sendResponseHeaders(TOO_MANY_REQUESTS.getStatus(), -1); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + server.start(); + try { + MockLog.awaitLogger( + () -> { + try { + runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { + BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + Reindexer.class, + new MockLog.SeenEventExpectation( + "Failed to close remote PIT (rejected) should be logged", + Reindexer.class.getCanonicalName(), + Level.WARN, + "Failed to close remote PIT (rejected)" + ) + ); + } finally { + server.stop(0); + } + } + + /** + * When TransportOpenPointInTimeAction fails in openPitAndExecute, the failure propagates to the listener. + * We use a custom Client that fails on OpenPointInTimeRequest; the listener receives that failure. + */ + public void testLocalReindexingRequestFailsToOpenPit() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final String expectedMessage = "open-pit-failure-" + randomAlphaOfLength(8); + final OpenPitFailingClient client = new OpenPitFailingClient(getTestName(), expectedMessage); + try { + final ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.generic()).thenReturn(DIRECT_EXECUTOR_SERVICE); + + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + null + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + final PlainActionFuture initFuture = new PlainActionFuture<>(); + reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l))); + initFuture.actionGet(); + + fail("expected listener to receive failure"); + } catch (Exception e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } finally { + client.shutdown(); + } + } + + /** + * When PIT search is enabled and the local PIT close fails, the failure is logged but the main listener + * still receives success. This verifies that close failures are handled gracefully and don't propagate. + */ + public void testLocalReindexingRequestFailsToClosePit() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final String closeFailureMessage = "close-pit-failure-" + randomAlphaOfLength(8); + final ClosePitFailingClient client = new ClosePitFailingClient(getTestName(), closeFailureMessage); + try { + final TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + null + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + MockLog.awaitLogger( + () -> { + final PlainActionFuture initFuture = new PlainActionFuture<>(); + reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l))); + final BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }, + Reindexer.class, + new MockLog.SeenEventExpectation( + "Failed to close local PIT should be logged", + Reindexer.class.getCanonicalName(), + Level.WARN, + "Failed to close local PIT" + ) + ); + } finally { + terminate(threadPool); + } + } finally { + client.shutdown(); + } + } + + /** + * Client that succeeds on OpenPointInTime and Search (empty results) but fails on ClosePointInTime. + * Used to verify that PIT close failures are logged but don't propagate to the main listener. + */ + private static final class ClosePitFailingClient extends NoOpClient { + private final String closeFailureMessage; + private final TestThreadPool threadPool; + + ClosePitFailingClient(String threadPoolName, String closeFailureMessage) { + super(new TestThreadPool(threadPoolName), TestProjectResolvers.DEFAULT_PROJECT_ONLY); + this.threadPool = (TestThreadPool) super.threadPool(); + this.closeFailureMessage = closeFailureMessage; + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (action == TransportOpenPointInTimeAction.TYPE && request instanceof OpenPointInTimeRequest) { + OpenPointInTimeResponse response = new OpenPointInTimeResponse(new BytesArray("pit-id"), 1, 1, 0, 0); + listener.onResponse((Response) response); + return; + } + if (action == TransportSearchAction.TYPE && request instanceof SearchRequest) { + SearchResponse response = SearchResponseUtils.successfulResponse( + SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0) + ); + listener.onResponse((Response) response); + return; + } + if (action == TransportClosePointInTimeAction.TYPE && request instanceof ClosePointInTimeRequest) { + listener.onFailure(new RuntimeException(closeFailureMessage)); + return; + } + super.doExecute(action, request, listener); + } + + void shutdown() { + terminate(threadPool); + } + } + + /** + * Client that fails when it receives an OpenPointInTimeRequest. Used to verify the local PIT path is taken. + */ + private static final class OpenPitFailingClient extends NoOpClient { + private final String failureMessage; + private final TestThreadPool threadPool; + + OpenPitFailingClient(String threadPoolName, String failureMessage) { + super( + new TestThreadPool(threadPoolName), + TestProjectResolvers.DEFAULT_PROJECT_ONLY + ); + this.threadPool = (TestThreadPool) super.threadPool(); + this.failureMessage = failureMessage; + } + + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (action == TransportOpenPointInTimeAction.TYPE && request instanceof OpenPointInTimeRequest) { + listener.onFailure(new RuntimeException(failureMessage)); + } else { + super.doExecute(action, request, listener); + } + } + + void shutdown() { + terminate(threadPool); + } + } + // --- helpers --- + private static final String REMOTE_PIT_TEST_VERSION_JSON = + "{\"version\":{\"number\":\"7.10.0\"},\"tagline\":\"You Know, for Search\"}"; + private static final String REMOTE_PIT_OPEN_RESPONSE = "{\"id\":\"c29tZXBpdGlk\"}"; + private static final String REMOTE_PIT_EMPTY_SEARCH_RESPONSE = + "{\"_scroll_id\":\"scroll1\",\"timed_out\":false,\"hits\":{\"total\":0,\"hits\":[]},\"_shards\":{\"total\":1,\"successful\":1,\"failed\":0}}"; + + /** + * Creates a MockHttpServer that handles the full remote PIT flow (version, open PIT, search, close PIT). + * For requests matching the predicate, the customHandler is used; otherwise standard success responses are returned. + */ + @SuppressForbidden(reason = "use http server for testing") + private HttpServer createRemotePitMockServer( + java.util.function.BiPredicate useCustomHandler, + java.util.function.Consumer customHandler + ) throws IOException { + HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + String path = exchange.getRequestURI().getPath(); + String method = exchange.getRequestMethod(); + if (useCustomHandler.test(path, method)) { + customHandler.accept(exchange); + } else if (path.equals("/") || path.isEmpty()) { + respondJson(exchange, 200, REMOTE_PIT_TEST_VERSION_JSON); + } else if (path.contains("_pit") && "POST".equals(method)) { + respondJson(exchange, 200, REMOTE_PIT_OPEN_RESPONSE); + } else if (path.contains("_search") && "POST".equals(method)) { + respondJson(exchange, 200, REMOTE_PIT_EMPTY_SEARCH_RESPONSE); + } else if (path.contains("_search/scroll") && "DELETE".equals(method)) { + exchange.sendResponseHeaders(200, -1); + } else { + exchange.sendResponseHeaders(404, -1); + } + exchange.close(); + }); + return server; + } + + private static void respondJson(HttpExchange exchange, int status, String json) throws IOException { + byte[] body = json.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + + /** + * Runs a remote PIT reindex test against a MockHttpServer. The server must already be started. + */ + @SuppressForbidden(reason = "use http server for testing") + private void runRemotePitTestWithMockServer( + HttpServer server, + java.util.function.Consumer requestConfigurer, + java.util.function.Consumer> assertions + ) { + BytesArray matchAll = new BytesArray("{\"match_all\":{}}"); + RemoteInfo remoteInfo = new RemoteInfo( + "http", + server.getAddress().getHostString(), + server.getAddress().getPort(), + null, + matchAll, + null, + null, + emptyMap(), + TimeValue.timeValueSeconds(5), + TimeValue.timeValueSeconds(5) + ); + + ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setRemoteInfo(remoteInfo); + request.setSlices(1); + requestConfigurer.accept(request); + + ClusterService clusterService = mock(ClusterService.class); + DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); + ReindexSslConfig sslConfig = new ReindexSslConfig(environment.settings(), environment, mock(ResourceWatcherService.class)); + + Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + mock(Client.class), + threadPool, + mock(ScriptService.class), + sslConfig, + null, + mock(TransportService.class), + null + ); + + BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + PlainActionFuture initFuture = new PlainActionFuture<>(); + reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, mock(Client.class), l))); + assertions.accept(initFuture); + } finally { + terminate(threadPool); + } + } + private BulkByScrollResponse reindexResponseWithBulkAndSearchFailures( final List bulkFailures, List searchFailures From e58bd07b61eecd904fbe2539f18f2099bef99cac Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 11:36:26 +0000 Subject: [PATCH 25/45] Spotless apply --- modules/reindex/build.gradle | 9 ++ .../elasticsearch/reindex/ReindexerTests.java | 136 +++++++++--------- .../remote/RemoteReindexingUtilsTests.java | 86 ++++------- 3 files changed, 108 insertions(+), 123 deletions(-) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 86eabc6e773d9..c0299c0ea3d3c 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -81,6 +81,15 @@ tasks.named("forbiddenPatterns").configure { exclude '**/*.p12' } +tasks.named('forbiddenApisTest').configure { + // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage + modifyBundledSignatures { bundledSignatures -> + bundledSignatures -= 'jdk-non-portable' + bundledSignatures += 'jdk-internal' + bundledSignatures + } +} + // Support for testing reindex-from-remote against old Elasticsearch versions configurations { oldesFixture diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 57cca377566cb..dfba9796f2ee0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -9,10 +9,17 @@ package org.elasticsearch.reindex; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import org.apache.logging.log4j.Level; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeResponse; @@ -21,7 +28,6 @@ import org.elasticsearch.action.search.TransportClosePointInTimeAction; import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; -import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -33,30 +39,25 @@ import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.PaginatedHitSource; import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.index.reindex.ResumeBulkByScrollRequest; import org.elasticsearch.index.reindex.ResumeBulkByScrollResponse; import org.elasticsearch.index.reindex.ResumeInfo; import org.elasticsearch.index.reindex.ResumeReindexAction; +import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.tasks.TaskId; -import org.apache.logging.log4j.Level; -import org.apache.lucene.search.TotalHits; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchResponseUtils; -import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLog; import org.elasticsearch.test.client.NoOpClient; @@ -66,9 +67,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpServer; - import java.io.IOException; import java.io.OutputStream; import java.net.InetAddress; @@ -83,9 +81,9 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE; +import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; -import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; @@ -388,29 +386,25 @@ public void testRemoteReindexingRequestFailsToOpenPit() throws Exception { public void testRemoteReindexingRequestFailsToClosePit() throws Exception { assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); - HttpServer server = createRemotePitMockServer( - (path, method) -> path.contains("_pit") && "DELETE".equals(method), - exchange -> { - try { - exchange.sendResponseHeaders(500, -1); - } catch (IOException e) { - throw new RuntimeException(e); - } + HttpServer server = createRemotePitMockServer((path, method) -> path.contains("_pit") && "DELETE".equals(method), exchange -> { + try { + exchange.sendResponseHeaders(500, -1); + } catch (IOException e) { + throw new RuntimeException(e); } - ); + }); server.start(); try { - MockLog.awaitLogger( - () -> { - try { - runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { - BulkByScrollResponse response = initFuture.actionGet(); - assertNotNull(response); - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, + MockLog.awaitLogger(() -> { + try { + runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { + BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, Reindexer.class, new MockLog.SeenEventExpectation( "Failed to close remote PIT should be logged", @@ -432,29 +426,25 @@ public void testRemoteReindexingRequestFailsToClosePit() throws Exception { public void testRemoteReindexingRequestFailsWhenClosePitIsRejected() throws Exception { assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); - HttpServer server = createRemotePitMockServer( - (path, method) -> path.contains("_pit") && "DELETE".equals(method), - exchange -> { - try { - exchange.sendResponseHeaders(TOO_MANY_REQUESTS.getStatus(), -1); - } catch (IOException e) { - throw new RuntimeException(e); - } + HttpServer server = createRemotePitMockServer((path, method) -> path.contains("_pit") && "DELETE".equals(method), exchange -> { + try { + exchange.sendResponseHeaders(TOO_MANY_REQUESTS.getStatus(), -1); + } catch (IOException e) { + throw new RuntimeException(e); } - ); + }); server.start(); try { - MockLog.awaitLogger( - () -> { - try { - runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { - BulkByScrollResponse response = initFuture.actionGet(); - assertNotNull(response); - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, + MockLog.awaitLogger(() -> { + try { + runRemotePitTestWithMockServer(server, request -> {}, initFuture -> { + BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, Reindexer.class, new MockLog.SeenEventExpectation( "Failed to close remote PIT (rejected) should be logged", @@ -580,13 +570,12 @@ public ExecutorService executor(String name) { false ); - MockLog.awaitLogger( - () -> { - final PlainActionFuture initFuture = new PlainActionFuture<>(); - reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l))); - final BulkByScrollResponse response = initFuture.actionGet(); - assertNotNull(response); - }, + MockLog.awaitLogger(() -> { + final PlainActionFuture initFuture = new PlainActionFuture<>(); + reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l))); + final BulkByScrollResponse response = initFuture.actionGet(); + assertNotNull(response); + }, Reindexer.class, new MockLog.SeenEventExpectation( "Failed to close local PIT should be logged", @@ -656,10 +645,7 @@ private static final class OpenPitFailingClient extends NoOpClient { private final TestThreadPool threadPool; OpenPitFailingClient(String threadPoolName, String failureMessage) { - super( - new TestThreadPool(threadPoolName), - TestProjectResolvers.DEFAULT_PROJECT_ONLY - ); + super(new TestThreadPool(threadPoolName), TestProjectResolvers.DEFAULT_PROJECT_ONLY); this.threadPool = (TestThreadPool) super.threadPool(); this.failureMessage = failureMessage; } @@ -684,11 +670,21 @@ void shutdown() { // --- helpers --- - private static final String REMOTE_PIT_TEST_VERSION_JSON = - "{\"version\":{\"number\":\"7.10.0\"},\"tagline\":\"You Know, for Search\"}"; + private static final String REMOTE_PIT_TEST_VERSION_JSON = "{\"version\":{\"number\":\"7.10.0\"},\"tagline\":\"You Know, for Search\"}"; private static final String REMOTE_PIT_OPEN_RESPONSE = "{\"id\":\"c29tZXBpdGlk\"}"; - private static final String REMOTE_PIT_EMPTY_SEARCH_RESPONSE = - "{\"_scroll_id\":\"scroll1\",\"timed_out\":false,\"hits\":{\"total\":0,\"hits\":[]},\"_shards\":{\"total\":1,\"successful\":1,\"failed\":0}}"; + private static final String REMOTE_PIT_EMPTY_SEARCH_RESPONSE = "{" + + "\"_scroll_id\":\"scroll1\"," + + "\"timed_out\":false," + + "\"hits\":{" + + "\"total\":0," + + "\"hits\":[]" + + "}," + + "\"_shards\":{" + + "\"total\":1," + + "\"successful\":1," + + "\"failed\":0" + + "}" + + "}"; /** * Creates a MockHttpServer that handles the full remote PIT flow (version, open PIT, search, close PIT). @@ -801,7 +797,11 @@ public ExecutorService executor(String name) { ); PlainActionFuture initFuture = new PlainActionFuture<>(); - reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, mock(Client.class), l))); + reindexer.initTask( + task, + request, + initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, mock(Client.class), l)) + ); assertions.accept(initFuture); } finally { terminate(threadPool); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 8459d3bbdbd79..1d123fba53ca3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -20,10 +20,10 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; import org.elasticsearch.client.Response; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.BackoffPolicy; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; @@ -452,14 +452,10 @@ public void testOpenPitSuccess() { RemoteReindexingUtils.openPit( new String[] { randomAlphaOfLength(between(1, 10)) }, TimeValue.timeValueMillis(between(1, 60000)), - RejectAwareActionListener.wrap( - pitId -> { - capturedPitId[0] = pitId; - success.set(true); - }, - e -> fail("unexpected failure"), - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(pitId -> { + capturedPitId[0] = pitId; + success.set(true); + }, e -> fail("unexpected failure"), e -> fail("unexpected rejection")), threadPool, client ); @@ -503,15 +499,11 @@ public void testOpenPitHttpErrorTriggersFailure() throws Exception { RemoteReindexingUtils.openPit( new String[] { randomAlphaOfLength(between(1, 10)) }, randomPositiveTimeValue(), - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof ElasticsearchStatusException); - assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); - failed.set(true); - }, - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); + failed.set(true); + }, e -> fail("unexpected rejection")), threadPool, client ); @@ -531,15 +523,11 @@ public void testOpenPitInvalidJsonTriggersFailure() { RemoteReindexingUtils.openPit( new String[] { randomAlphaOfLength(between(1, 10)) }, randomPositiveTimeValue(), - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof ElasticsearchException); - assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); - failed.set(true); - }, - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof ElasticsearchException); + assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); + failed.set(true); + }, e -> fail("unexpected rejection")), threadPool, client ); @@ -559,15 +547,11 @@ public void testOpenPitMissingIdFieldTriggersFailure() { RemoteReindexingUtils.openPit( new String[] { randomAlphaOfLength(between(1, 10)) }, randomPositiveTimeValue(), - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof IllegalArgumentException); - assertThat(e.getMessage(), containsString("open point-in-time response must contain [id] field")); - failed.set(true); - }, - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof IllegalArgumentException); + assertThat(e.getMessage(), containsString("open point-in-time response must contain [id] field")); + failed.set(true); + }, e -> fail("unexpected rejection")), threadPool, client ); @@ -628,15 +612,11 @@ public void testClosePitHttpErrorTriggersFailure() throws Exception { AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.closePit( new BytesArray(randomByteArrayOfLength(between(1, 32))), - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof ElasticsearchStatusException); - assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); - failed.set(true); - }, - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof ElasticsearchStatusException); + assertEquals(statusCode, ((ElasticsearchStatusException) e).status().getStatus()); + failed.set(true); + }, e -> fail("unexpected rejection")), threadPool, client ); @@ -655,15 +635,11 @@ public void testClosePitInvalidJsonTriggersFailure() { AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.closePit( new BytesArray(randomByteArrayOfLength(between(1, 32))), - RejectAwareActionListener.wrap( - v -> fail("unexpected success"), - e -> { - assertTrue(e instanceof ElasticsearchException); - assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); - failed.set(true); - }, - e -> fail("unexpected rejection") - ), + RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { + assertTrue(e instanceof ElasticsearchException); + assertThat(e.getMessage(), containsString("remote is likely not an Elasticsearch instance")); + failed.set(true); + }, e -> fail("unexpected rejection")), threadPool, client ); From d1c9a09aa669b9f42464492708b2714a926c38b8 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 12:30:29 +0000 Subject: [PATCH 26/45] Merge main --- .../org/elasticsearch/reindex/Reindexer.java | 147 ++++++++++-------- ...natedSearchParallelizationHelperTests.java | 4 +- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index aec8ef1dcccc4..e48442b3f3a7e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -91,6 +91,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -173,8 +174,8 @@ else if (isRemote) { } /** - * Creates the worker action that runs the reindex. The listener is invoked on success or failure; for PIT paths, - * the listener should include runAfter logic to close the PIT. + * Creates the worker action that runs the reindex. + * When PIT is used, the listener should include runAfter logic to close the PIT. */ private Consumer createWorkerAction( BulkByScrollTask task, @@ -211,14 +212,7 @@ private Consumer createWorkerAction( } /** - * Returns the keep-alive duration for PIT. Uses the request's scroll time when set, otherwise defaults to 5 minutes. - */ - private static TimeValue pitKeepAlive(ReindexRequest request) { - return request.getScrollTime() != null ? request.getScrollTime() : TimeValue.timeValueMinutes(5); - } - - /** - * Runs the sliced action using scroll. + * Runs the sliced action */ private void executePaginatedSearch( BulkByScrollTask task, @@ -239,6 +233,13 @@ private void executePaginatedSearch( ); } + /** + * Returns the keep-alive duration for PIT. Uses the request's scroll time when set, otherwise defaults to 5 minutes. + */ + private static TimeValue pitKeepAlive(ReindexRequest request) { + return request.getScrollTime() != null ? request.getScrollTime() : TimeValue.timeValueMinutes(5); + } + /** * Opens a PIT on the local cluster, runs the sliced action, and closes the PIT when done. */ @@ -255,36 +256,46 @@ private void openPitAndExecute( startTime, false ); + SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); - OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices).indicesOptions(searchRequest.indicesOptions()) + OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices) + .indicesOptions(searchRequest.indicesOptions()) .keepAlive(pitKeepAlive(request)); - client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { - BytesReference pitId = pitResponse.getPointInTimeId(); - ActionListener listenerWithClosePit = ActionListener.runAfter( - l, - () -> client.execute( - TransportClosePointInTimeAction.TYPE, - new ClosePointInTimeRequest(pitId), - ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) - ) - ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - false, - true - ); - executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); - })); + + // NB this is a local request, so we call the TransportAction rather than issuing a REST call + client.execute( + TransportOpenPointInTimeAction.TYPE, + pitRequest, + listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { + BytesReference pitId = pitResponse.getPointInTimeId(); + ActionListener listenerWithClosePit = ActionListener.runAfter( + l, + () -> client.execute( + TransportClosePointInTimeAction.TYPE, + new ClosePointInTimeRequest(pitId), + ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) + ) + ); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + false, + true + ); + // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); + }) + ); } /** - * Looks up the remote cluster version when reindexing from a remote source. If the remote supports PIT (7.10.0+), - * opens PIT, runs the sliced action, and closes PIT when done. Otherwise uses scroll. + * Looks up the remote cluster version when reindexing from a remote source. + * If the remote supports PIT (7.10.0+), this opens a PIT, runs the sliced action, and closes the PIT when done. + * Otherwise, the default search method is scroll. * The RestClient used for lookup (and PIT open/close when applicable) is closed after completion. */ private void lookupRemoteVersionAndExecute( @@ -295,12 +306,14 @@ private void lookupRemoteVersionAndExecute( Consumer workerAction, long startTime ) { + // TODO - Update the metrics integration tests? // Wrap with metrics so failures before the worker runs (version lookup, PIT open) are recorded final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( listenerWithRelocations, startTime, true ); + RemoteInfo remoteInfo = request.getRemoteInfo(); assert reindexSslConfig != null : "Reindex ssl config must be set"; RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); @@ -310,7 +323,9 @@ public void onResponse(Version version) { boolean canUsePit = version.onOrAfter(Version.V_7_10_0); if (canUsePit) { openRemotePitAndExecute(task, request, bulkClient, listenerWithMetrics, restClient, version, startTime); - } else { + } + // Default to scroll-based search + else { closeRestClientAndRun( restClient, () -> executePaginatedSearch(task, request, listenerWithRelocations, workerAction, version) @@ -328,6 +343,7 @@ public void onRejection(Exception e) { closeRestClientAndRun(restClient, () -> listenerWithMetrics.onFailure(e)); } }; + RemoteReindexingUtils.lookupRemoteVersionWithRetries( logger, exponentialBackoff(request.getRetryBackoffInitialTime(), request.getMaxRetries()), @@ -339,8 +355,6 @@ public void onRejection(Exception e) { ); } - // TODO - Retry logic - /** * Opens a PIT on the remote cluster, runs the sliced action, and closes the PIT when done. * The RestClient is closed after the PIT is closed. @@ -355,31 +369,40 @@ private void openRemotePitAndExecute( long startTime ) { String[] indices = request.getSearchRequest().indices(); - openPit(indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { - ActionListener listenerWithClosePit = ActionListener.runAfter( - listenerWithRelocations, - () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { - logger.warn("Failed to close remote PIT", e); - closeRestClientAndRun(restClient, () -> {}); - }, e -> { - logger.warn("Failed to close remote PIT (rejected)", e); - closeRestClientAndRun(restClient, () -> {}); - }), threadPool, restClient) - ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - true, - true - ); - executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); - }, - e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), - e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) - ), threadPool, restClient); + // Sends a REST request to the remote node to open a PIT + openPit( + indices, + pitKeepAlive(request), + RejectAwareActionListener.wrap( + pitId -> { + ActionListener listenerWithClosePit = ActionListener.runAfter( + listenerWithRelocations, + () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { + logger.warn("Failed to close remote PIT", e); + closeRestClientAndRun(restClient, () -> {}); + }, e -> { + logger.warn("Failed to close remote PIT (rejected)", e); + closeRestClientAndRun(restClient, () -> {}); + }), threadPool, restClient) + ); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + true, + true + ); + // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); + }, + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) + ), + threadPool, + restClient + ); } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index ff4b842188b40..c7d39d96132bf 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -38,12 +38,10 @@ import static org.elasticsearch.reindex.BulkByPaginatedSearchParallelizationHelper.sliceIntoSubRequests; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertThat; public class BulkByPaginatedSearchParallelizationHelperTests extends ESTestCase { From 3af6d7cabc13a3e6a54e8e0f2a2ff77b2d55b3f9 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 12:57:00 +0000 Subject: [PATCH 27/45] Fix tests post merge --- .../org/elasticsearch/reindex/Reindexer.java | 110 ++++++++---------- .../elasticsearch/reindex/ReindexerTests.java | 9 +- 2 files changed, 54 insertions(+), 65 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index e48442b3f3a7e..d044b70c53032 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -259,37 +259,32 @@ private void openPitAndExecute( SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); - OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices) - .indicesOptions(searchRequest.indicesOptions()) + OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices).indicesOptions(searchRequest.indicesOptions()) .keepAlive(pitKeepAlive(request)); // NB this is a local request, so we call the TransportAction rather than issuing a REST call - client.execute( - TransportOpenPointInTimeAction.TYPE, - pitRequest, - listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { - BytesReference pitId = pitResponse.getPointInTimeId(); - ActionListener listenerWithClosePit = ActionListener.runAfter( - l, - () -> client.execute( - TransportClosePointInTimeAction.TYPE, - new ClosePointInTimeRequest(pitId), - ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) - ) - ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - false, - true - ); - // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used - executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); - }) - ); + client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { + BytesReference pitId = pitResponse.getPointInTimeId(); + ActionListener listenerWithClosePit = ActionListener.runAfter( + l, + () -> client.execute( + TransportClosePointInTimeAction.TYPE, + new ClosePointInTimeRequest(pitId), + ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) + ) + ); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + false, + true + ); + // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); + })); } /** @@ -370,39 +365,32 @@ private void openRemotePitAndExecute( ) { String[] indices = request.getSearchRequest().indices(); // Sends a REST request to the remote node to open a PIT - openPit( - indices, - pitKeepAlive(request), - RejectAwareActionListener.wrap( - pitId -> { - ActionListener listenerWithClosePit = ActionListener.runAfter( - listenerWithRelocations, - () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { - logger.warn("Failed to close remote PIT", e); - closeRestClientAndRun(restClient, () -> {}); - }, e -> { - logger.warn("Failed to close remote PIT (rejected)", e); - closeRestClientAndRun(restClient, () -> {}); - }), threadPool, restClient) - ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - true, - true - ); - // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used - executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); - }, - e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), - e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) - ), - threadPool, - restClient - ); + openPit(indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { + ActionListener listenerWithClosePit = ActionListener.runAfter( + listenerWithRelocations, + () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { + logger.warn("Failed to close remote PIT", e); + closeRestClientAndRun(restClient, () -> {}); + }, e -> { + logger.warn("Failed to close remote PIT (rejected)", e); + closeRestClientAndRun(restClient, () -> {}); + }), threadPool, restClient) + ); + Consumer workerActionWithClosePit = createWorkerAction( + task, + request, + bulkClient, + listenerWithClosePit, + startTime, + true, + true + ); + // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used + executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); + }, + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)), + e -> closeRestClientAndRun(restClient, () -> listenerWithRelocations.onFailure(e)) + ), threadPool, restClient); } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index dfba9796f2ee0..75e00f45f22aa 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.Level; import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -488,7 +489,7 @@ public void testLocalReindexingRequestFailsToOpenPit() { mock(ReindexSslConfig.class), null, mock(TransportService.class), - null + mock(ReindexRelocationNodePicker.class) ); final ReindexRequest request = new ReindexRequest(); @@ -512,7 +513,7 @@ public void testLocalReindexingRequestFailsToOpenPit() { fail("expected listener to receive failure"); } catch (Exception e) { - assertThat(e.getMessage(), containsString(expectedMessage)); + assertThat(ExceptionsHelper.unwrapCause(e).getMessage(), containsString(expectedMessage)); } finally { client.shutdown(); } @@ -552,7 +553,7 @@ public ExecutorService executor(String name) { mock(ReindexSslConfig.class), null, mock(TransportService.class), - null + mock(ReindexRelocationNodePicker.class) ); final ReindexRequest request = new ReindexRequest(); @@ -783,7 +784,7 @@ public ExecutorService executor(String name) { sslConfig, null, mock(TransportService.class), - null + mock(ReindexRelocationNodePicker.class) ); BulkByScrollTask task = new BulkByScrollTask( From 1dbc15f645d00e321a47c68db1e05f0c49426d11 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 13:04:46 +0000 Subject: [PATCH 28/45] Nits --- .../src/main/java/org/elasticsearch/reindex/Reindexer.java | 1 + .../reindex/BulkByPaginatedSearchParallelizationHelperTests.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index d044b70c53032..077f3c53ae3f1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -235,6 +235,7 @@ private void executePaginatedSearch( /** * Returns the keep-alive duration for PIT. Uses the request's scroll time when set, otherwise defaults to 5 minutes. + * TODO - https://github.com/elastic/elasticsearch-team/issues/2334 */ private static TimeValue pitKeepAlive(ReindexRequest request) { return request.getScrollTime() != null ? request.getScrollTime() : TimeValue.timeValueMinutes(5); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index c7d39d96132bf..dad4e9885e0e4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -41,7 +41,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; public class BulkByPaginatedSearchParallelizationHelperTests extends ESTestCase { From 57ac7a18eb68445ac0505b009a6125cfd32a41ef Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 13:11:44 +0000 Subject: [PATCH 29/45] Remove FQ imports --- .../org/elasticsearch/reindex/ReindexerTests.java | 10 ++++++---- .../remote/RemoteReindexingUtilsTests.java | 15 ++++++++------- .../remote/RemoteRequestBuildersTests.java | 7 ++++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 75e00f45f22aa..cf4cba48b1136 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -78,6 +78,8 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.function.BiPredicate; +import java.util.function.Consumer; import java.util.concurrent.atomic.AtomicInteger; import static java.util.Collections.emptyMap; @@ -693,8 +695,8 @@ void shutdown() { */ @SuppressForbidden(reason = "use http server for testing") private HttpServer createRemotePitMockServer( - java.util.function.BiPredicate useCustomHandler, - java.util.function.Consumer customHandler + BiPredicate useCustomHandler, + Consumer customHandler ) throws IOException { HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); server.createContext("/", exchange -> { @@ -733,8 +735,8 @@ private static void respondJson(HttpExchange exchange, int status, String json) @SuppressForbidden(reason = "use http server for testing") private void runRemotePitTestWithMockServer( HttpServer server, - java.util.function.Consumer requestConfigurer, - java.util.function.Consumer> assertions + Consumer requestConfigurer, + Consumer> assertions ) { BytesArray matchAll = new BytesArray("{\"match_all\":{}}"); RemoteInfo remoteInfo = new RemoteInfo( diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 9482f51fcc56a..b34024626ab0e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -42,6 +42,7 @@ import java.io.IOException; import java.net.URL; +import java.util.Base64; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; @@ -437,7 +438,7 @@ public void testLookupRemoteVersionWithRetriesSucceedsOnFirstCall() throws Excep */ public void testOpenPitSuccess() { byte[] pitIdBytes = randomByteArrayOfLength(between(1, 64)); - String base64Id = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + String base64Id = Base64.getUrlEncoder().encodeToString(pitIdBytes); String json = "{\"id\":\"" + base64Id + "\"}"; Response response = mock(Response.class); when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); @@ -463,7 +464,7 @@ public void testOpenPitSuccess() { * Verifies that openPit invokes onRejection when the remote returns HTTP 429. */ public void testOpenPitTooManyRequestsTriggersRejection() throws Exception { - mockFailure(new org.elasticsearch.client.ResponseException(rejectionResponse429())); + mockFailure(new ResponseException(rejectionResponse429())); AtomicBoolean rejected = new AtomicBoolean(false); RemoteReindexingUtils.openPit( @@ -481,7 +482,7 @@ public void testOpenPitTooManyRequestsTriggersRejection() throws Exception { */ public void testOpenPitHttpErrorTriggersFailure() throws Exception { int statusCode = randomFrom(RestStatus.BAD_REQUEST, RestStatus.NOT_FOUND, RestStatus.INTERNAL_SERVER_ERROR).getStatus(); - org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + StatusLine statusLine = mock(StatusLine.class); when(statusLine.getStatusCode()).thenReturn(statusCode); Response response = mock(Response.class); when(response.getStatusLine()).thenReturn(statusLine); @@ -489,7 +490,7 @@ public void testOpenPitHttpErrorTriggersFailure() throws Exception { RequestLine requestLine = mock(RequestLine.class); when(requestLine.getMethod()).thenReturn("POST"); when(response.getRequestLine()).thenReturn(requestLine); - mockFailure(new org.elasticsearch.client.ResponseException(response)); + mockFailure(new ResponseException(response)); AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.openPit( @@ -578,7 +579,7 @@ public void testClosePitSuccess() { * Verifies that closePit invokes onRejection when the remote returns HTTP 429. */ public void testClosePitTooManyRequestsTriggersRejection() throws Exception { - mockFailure(new org.elasticsearch.client.ResponseException(rejectionResponse429())); + mockFailure(new ResponseException(rejectionResponse429())); AtomicBoolean rejected = new AtomicBoolean(false); RemoteReindexingUtils.closePit( @@ -595,7 +596,7 @@ public void testClosePitTooManyRequestsTriggersRejection() throws Exception { */ public void testClosePitHttpErrorTriggersFailure() throws Exception { int statusCode = randomFrom(RestStatus.BAD_REQUEST, RestStatus.NOT_FOUND, RestStatus.INTERNAL_SERVER_ERROR).getStatus(); - org.apache.http.StatusLine statusLine = mock(org.apache.http.StatusLine.class); + StatusLine statusLine = mock(StatusLine.class); when(statusLine.getStatusCode()).thenReturn(statusCode); Response response = mock(Response.class); when(response.getStatusLine()).thenReturn(statusLine); @@ -603,7 +604,7 @@ public void testClosePitHttpErrorTriggersFailure() throws Exception { RequestLine requestLine = mock(RequestLine.class); when(requestLine.getMethod()).thenReturn("DELETE"); when(response.getRequestLine()).thenReturn(requestLine); - mockFailure(new org.elasticsearch.client.ResponseException(response)); + mockFailure(new ResponseException(response)); AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.closePit( diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java index 505295f91ce08..6340aae297aff 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Map; import static org.elasticsearch.core.TimeValue.timeValueMillis; @@ -415,7 +416,7 @@ public void testClosePitRequestStructure() throws IOException { assertThat(request.getEntity(), not(nullValue())); assertEquals(ContentType.APPLICATION_JSON.toString(), request.getEntity().getContentType().getValue()); String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); - String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + String expectedId = Base64.getUrlEncoder().encodeToString(pitIdBytes); assertThat(body, containsString("\"id\":\"" + expectedId + "\"")); } @@ -427,7 +428,7 @@ public void testClosePitEncodesBinaryPitId() throws IOException { BytesReference pitId = new BytesArray(pitIdBytes); Request request = closePit(pitId); String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); - String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdBytes); + String expectedId = Base64.getUrlEncoder().encodeToString(pitIdBytes); assertThat(body, containsString("\"id\":\"" + expectedId + "\"")); } @@ -439,7 +440,7 @@ public void testClosePitProducesValidJson() throws IOException { BytesReference pitId = new BytesArray(pitIdStr.getBytes(StandardCharsets.UTF_8)); Request request = closePit(pitId); String body = Streams.copyToString(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)); - String expectedId = java.util.Base64.getUrlEncoder().encodeToString(pitIdStr.getBytes(StandardCharsets.UTF_8)); + String expectedId = Base64.getUrlEncoder().encodeToString(pitIdStr.getBytes(StandardCharsets.UTF_8)); assertThat(body, containsString("\"id\"")); assertThat(body, containsString(expectedId)); assertThat(body.trim(), startsWith("{")); From 57599878d9db928c5da4305174ba1691920ab239 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 14:53:33 +0000 Subject: [PATCH 30/45] Add metric IT --- .../index/reindex/ReindexPluginMetricsIT.java | 123 ++++++++++++++++++ .../elasticsearch/reindex/ReindexerTests.java | 8 +- 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java index dcb4c92a0d4c9..6a18127a64d71 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java @@ -10,13 +10,16 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.reindex.BulkIndexByScrollResponseMatcher; import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.reindex.Reindexer; import org.elasticsearch.reindex.TransportReindexAction; import org.elasticsearch.rest.root.MainRestPlugin; import org.elasticsearch.search.sort.SortOrder; @@ -42,6 +45,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertNull; @ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) public class ReindexPluginMetricsIT extends ESIntegTestCase { @@ -221,6 +226,124 @@ public void testReindexMetricsWithBulkFailure() throws Exception { }); } + /** + * Verifies that remote reindex metrics record failures when the remote version lookup fails + * (e.g. connection refused, host unreachable). + */ + public void testRemoteReindexVersionLookupFailureMetrics() throws Exception { + assumeTrue("PIT search must be enabled for remote version lookup path", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final String dataNodeName = internalCluster().startNode(); + + // Use an invalid host so version lookup fails during connection + RemoteInfo invalidRemote = new RemoteInfo( + "http", + "invalid.invalid", + 9200, + null, + new BytesArray("{\"match_all\":{}}"), + null, + null, + Map.of(), + TimeValue.timeValueMillis(100), + TimeValue.timeValueMillis(100) + ); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + expectThrows(Exception.class, () -> reindex().source("source").setRemoteInfo(invalidRemote).destination("dest").get()); + + assertBusy(() -> { + testTelemetryPlugin.collect(); + assertThat(testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM).size(), equalTo(1)); + List completions = testTelemetryPlugin.getLongCounterMeasurement(REINDEX_COMPLETION_COUNTER); + assertThat(completions.size(), equalTo(1)); + assertThat(completions.getFirst().attributes().get(ATTRIBUTE_NAME_ERROR_TYPE), notNullValue()); + assertThat(completions.getFirst().attributes().get(ATTRIBUTE_NAME_SOURCE), equalTo(ATTRIBUTE_VALUE_SOURCE_REMOTE)); + }); + } + + /** + * Verifies that no reindex metrics are recorded when validation fails before the {@link Reindexer} runs + * (e.g. source index does not exist). + */ + public void testLocalReindexValidationFailureNoMetrics() { + final String dataNodeName = internalCluster().startNode(); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + expectThrows(Exception.class, () -> reindex().source("non_existent_index").destination("dest").get()); + + testTelemetryPlugin.collect(); + assertThat(testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM).size(), equalTo(0)); + assertThat(testTelemetryPlugin.getLongCounterMeasurement(REINDEX_COMPLETION_COUNTER).size(), equalTo(0)); + } + + /** + * Verifies that local reindex metrics record failures when PIT open fails (e.g. source index is closed). + */ + public void testLocalReindexPitOpenFailureMetrics() throws Exception { + assumeTrue("PIT search must be enabled for local PIT path", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final String dataNodeName = internalCluster().startNode(); + + // Create and close the source index so PIT open fails (validation passes because index exists) + indexRandom(true, prepareIndex("source").setId("1").setSource("foo", "a")); + indicesAdmin().prepareClose("source").get(); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Use STRICT_EXPAND_OPEN_CLOSED so validation resolves the closed index; PIT open will still fail + ReindexRequestBuilder builder = reindex().source("source").destination("dest"); + builder.source().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_CLOSED); + expectThrows(Exception.class, () -> builder.get()); + + assertBusy(() -> { + testTelemetryPlugin.collect(); + assertThat(testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM).size(), equalTo(1)); + List completions = testTelemetryPlugin.getLongCounterMeasurement(REINDEX_COMPLETION_COUNTER); + assertThat(completions.size(), equalTo(1)); + assertThat(completions.getFirst().attributes().get(ATTRIBUTE_NAME_ERROR_TYPE), notNullValue()); + assertThat(completions.getFirst().attributes().get(ATTRIBUTE_NAME_SOURCE), equalTo(ATTRIBUTE_VALUE_SOURCE_LOCAL)); + }); + } + + /** + * Verifies reindex metrics for a successful local reindex. + * Uses the scroll path when PIT is disabled, or the PIT path when PIT is enabled. + */ + public void testLocalReindexMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom(true, prepareIndex("source").setId("1").setSource("foo", "a"), prepareIndex("source").setId("2").setSource("foo", "b")); + assertHitCount(prepareSearch("source").setSize(0), 2); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + reindex().source("source").destination("dest").get(); + + assertBusy(() -> { + testTelemetryPlugin.collect(); + assertThat(testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM).size(), equalTo(1)); + List completions = testTelemetryPlugin.getLongCounterMeasurement(REINDEX_COMPLETION_COUNTER); + assertThat(completions.size(), equalTo(1)); + assertNull(completions.getFirst().attributes().get(ATTRIBUTE_NAME_ERROR_TYPE)); + assertThat(completions.getFirst().attributes().get(ATTRIBUTE_NAME_SOURCE), equalTo(ATTRIBUTE_VALUE_SOURCE_LOCAL)); + }); + } + public void testDeleteByQueryMetrics() throws Exception { final String dataNodeName = internalCluster().startNode(); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index cf4cba48b1136..85f98827f2f32 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -78,9 +78,9 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; import java.util.function.Consumer; -import java.util.concurrent.atomic.AtomicInteger; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE; @@ -694,10 +694,8 @@ void shutdown() { * For requests matching the predicate, the customHandler is used; otherwise standard success responses are returned. */ @SuppressForbidden(reason = "use http server for testing") - private HttpServer createRemotePitMockServer( - BiPredicate useCustomHandler, - Consumer customHandler - ) throws IOException { + private HttpServer createRemotePitMockServer(BiPredicate useCustomHandler, Consumer customHandler) + throws IOException { HttpServer server = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); server.createContext("/", exchange -> { String path = exchange.getRequestURI().getPath(); From aaed26727e0edc5f9f223c532e0e547a0296f154 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 14:53:55 +0000 Subject: [PATCH 31/45] Remove TODO --- .../src/main/java/org/elasticsearch/reindex/Reindexer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 077f3c53ae3f1..f8af3db419cb4 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -302,7 +302,6 @@ private void lookupRemoteVersionAndExecute( Consumer workerAction, long startTime ) { - // TODO - Update the metrics integration tests? // Wrap with metrics so failures before the worker runs (version lookup, PIT open) are recorded final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( listenerWithRelocations, From ed69ab63be4c1d3a583dd2d8d6ac55bf5aaa3436 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Fri, 27 Feb 2026 14:54:46 +0000 Subject: [PATCH 32/45] Unused imports --- .../org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java | 1 - .../BulkByPaginatedSearchParallelizationHelperTests.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java index 6a18127a64d71..14e8a82f727a9 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java @@ -46,7 +46,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertNull; @ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) public class ReindexPluginMetricsIT extends ESIntegTestCase { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index dad4e9885e0e4..58c0a063f61fd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -58,7 +58,7 @@ public void tearDownTaskManager() { terminate(threadPool); } - public void testSliceIntoSubRequests() throws IOException { + public void testSliceIntoSubRequests() { SearchRequest searchRequest = randomSearchRequest( () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null) ); From d56a445cb4bff5166cb5dd5321b8420440b0c5e4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 27 Feb 2026 15:04:42 +0000 Subject: [PATCH 33/45] [CI] Auto commit changes from spotless --- .../reindex/BulkByPaginatedSearchParallelizationHelperTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index 58c0a063f61fd..06b231709d372 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -29,7 +29,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.Collections; import java.util.concurrent.atomic.AtomicReference; From c742468c16a079a6f032616ba1e4e867c54771b9 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 4 Mar 2026 13:11:16 +0000 Subject: [PATCH 34/45] Fix parser and remove build.gradle changes --- modules/reindex/build.gradle | 9 --- .../reindex/remote/RemoteResponseParsers.java | 25 +++----- ...natedSearchParallelizationHelperTests.java | 1 - .../elasticsearch/reindex/ReindexerTests.java | 1 + .../remote/RemoteReindexingUtilsTests.java | 15 ++++- .../remote/RemoteResponseParsersTests.java | 61 ++++++++++++++----- 6 files changed, 70 insertions(+), 42 deletions(-) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index c0299c0ea3d3c..86eabc6e773d9 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -81,15 +81,6 @@ tasks.named("forbiddenPatterns").configure { exclude '**/*.p12' } -tasks.named('forbiddenApisTest').configure { - // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage - modifyBundledSignatures { bundledSignatures -> - bundledSignatures -= 'jdk-non-portable' - bundledSignatures += 'jdk-internal' - bundledSignatures - } -} - // Support for testing reindex-from-remote against old Elasticsearch versions configurations { oldesFixture diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java index 7468f9599231c..b03d6c9e59b06 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java @@ -274,27 +274,20 @@ public void setCausedBy(Throwable causedBy) { /** * Parser for the open point-in-time response. Returns the PIT id as {@link BytesReference}. */ - public static final BiFunction OPEN_PIT_PARSER = (p, xContentType) -> { - try { - String id = null; - if (p.nextToken() != XContentParser.Token.START_OBJECT) { - throw new IllegalArgumentException("open point-in-time response must be an object"); - } - while (p.nextToken() != XContentParser.Token.END_OBJECT) { - if (p.currentToken() == XContentParser.Token.FIELD_NAME && "id".equals(p.currentName())) { - p.nextToken(); - id = p.text(); - break; - } - } + public static final ConstructingObjectParser OPEN_PIT_PARSER = new ConstructingObjectParser<>( + "open_pit_response", + true, + a -> { + String id = (String) a[0]; if (id == null || id.isEmpty()) { throw new IllegalArgumentException("open point-in-time response must contain [id] field"); } return new BytesArray(Base64.getUrlDecoder().decode(id)); - } catch (IOException e) { - throw new RuntimeException("Failed to parse open point-in-time response", e); } - }; + ); + static { + OPEN_PIT_PARSER.declareString(optionalConstructorArg(), new ParseField("id")); + } /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java index 58c0a063f61fd..06b231709d372 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByPaginatedSearchParallelizationHelperTests.java @@ -29,7 +29,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.Collections; import java.util.concurrent.atomic.AtomicReference; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 85f98827f2f32..a3e2d34a26e29 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -101,6 +101,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +@SuppressForbidden(reason = "use a http server") public class ReindexerTests extends ESTestCase { public void testWrapWithMetricsSuccess() { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index b34024626ab0e..a8920614143d3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -62,6 +62,17 @@ public class RemoteReindexingUtilsTests extends ESTestCase { private static final Logger logger = LogManager.getLogger(RemoteReindexingUtilsTests.class); + private static String getAllExceptionMessages(Throwable t) { + StringBuilder sb = new StringBuilder(); + while (t != null) { + if (t.getMessage() != null) { + sb.append(t.getMessage()).append(" "); + } + t = t.getCause(); + } + return sb.toString(); + } + private ThreadPool threadPool; private RestClient client; @@ -545,8 +556,8 @@ public void testOpenPitMissingIdFieldTriggersFailure() { new String[] { randomAlphaOfLength(between(1, 10)) }, randomPositiveTimeValue(), RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { - assertTrue(e instanceof IllegalArgumentException); - assertThat(e.getMessage(), containsString("open point-in-time response must contain [id] field")); + assertTrue(e instanceof ElasticsearchException); + assertThat(getAllExceptionMessages(e), containsString("open point-in-time response must contain [id] field")); failed.set(true); }, e -> fail("unexpected rejection")), threadPool, diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java index 1b54c7c6638c5..05cba4c635e28 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.reindex.remote; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; @@ -16,6 +17,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.hamcrest.Matchers; @@ -77,11 +79,11 @@ public void testOpenPitParserValidResponse() throws IOException { public void testOpenPitParserMissingId() throws IOException { XContentBuilder builder = jsonBuilder().startObject().endObject(); try (XContentParser parser = createParser(builder)) { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) + Exception e = expectThrows(Exception.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat( + ExceptionsHelper.unwrapCause(e).getMessage(), + Matchers.containsString("Failed to build [open_pit_response] after last required field arrived") ); - assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must contain [id] field")); } } @@ -91,11 +93,8 @@ public void testOpenPitParserMissingId() throws IOException { public void testOpenPitParserEmptyId() throws IOException { XContentBuilder builder = jsonBuilder().startObject().field("id", "").endObject(); try (XContentParser parser = createParser(builder)) { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) - ); - assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must contain [id] field")); + Exception e = expectThrows(Exception.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat(ExceptionsHelper.unwrapCause(e).getMessage(), Matchers.containsString("failed to parse field [id]")); } } @@ -105,11 +104,44 @@ public void testOpenPitParserEmptyId() throws IOException { public void testOpenPitParserNotAnObject() throws IOException { XContentBuilder builder = jsonBuilder().startArray().value("a").endArray(); try (XContentParser parser = createParser(builder)) { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON) + XContentParseException e = expectThrows(XContentParseException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat(e.getMessage(), Matchers.containsString("Expected START_OBJECT")); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the response is a primitive value rather than an object. + */ + public void testOpenPitParserNotAnObjectWithPrimitive() throws IOException { + XContentBuilder builder = randomFrom(jsonBuilder().value("bare_string"), jsonBuilder().value(42), jsonBuilder().value(true)); + try (XContentParser parser = createParser(builder)) { + XContentParseException e = expectThrows(XContentParseException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat(e.getMessage(), Matchers.containsString("Expected START_OBJECT")); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the id field has the wrong type (e.g. number instead of string). + */ + public void testOpenPitParserIdWrongType() throws IOException { + XContentBuilder builder = jsonBuilder().startObject().field("id", 12345).endObject(); + try (XContentParser parser = createParser(builder)) { + XContentParseException e = expectThrows(XContentParseException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat(e.getMessage(), Matchers.anyOf(Matchers.containsString("id doesn't support values of type"))); + } + } + + /** + * Verifies that OPEN_PIT_PARSER throws when the id field is explicitly null. + */ + public void testOpenPitParserNullId() throws IOException { + XContentBuilder builder = jsonBuilder().startObject().nullField("id").endObject(); + try (XContentParser parser = createParser(builder)) { + Exception e = expectThrows(Exception.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat( + ExceptionsHelper.unwrapCause(e).getMessage(), + Matchers.containsString("id doesn't support values of type: VALUE_NULL") ); - assertThat(e.getMessage(), Matchers.containsString("open point-in-time response must be an object")); } } @@ -120,7 +152,8 @@ public void testOpenPitParserInvalidBase64() throws IOException { String invalidBase64 = randomAlphaOfLength(between(1, 20)) + "!!!"; XContentBuilder builder = jsonBuilder().startObject().field("id", invalidBase64).endObject(); try (XContentParser parser = createParser(builder)) { - expectThrows(IllegalArgumentException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + XContentParseException e = expectThrows(XContentParseException.class, () -> OPEN_PIT_PARSER.apply(parser, XContentType.JSON)); + assertThat(e.getMessage(), Matchers.containsString("failed to parse field [id]")); } } } From abb675aa41d3e860359316f0d7146acf09279a9b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 4 Mar 2026 13:23:39 +0000 Subject: [PATCH 35/45] [CI] Auto commit changes from spotless --- .../src/main/java/org/elasticsearch/reindex/Reindexer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index b5bd5eec40a7a..6e07824731723 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -105,7 +105,6 @@ import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.closePit; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.openPit; -import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE; public class Reindexer { From 437e6d141a467ef70a8012e6ab654eb4517429db Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 11:38:31 +0000 Subject: [PATCH 36/45] Add assertions to openPit --- .../org/elasticsearch/reindex/Reindexer.java | 5 +- .../reindex/remote/RemoteReindexingUtils.java | 7 ++ .../elasticsearch/reindex/ReindexerTests.java | 15 +++- .../remote/RemoteReindexingUtilsTests.java | 76 +++++++++++++++++-- 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index b5bd5eec40a7a..2a1a752c959a0 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -372,9 +372,10 @@ private void openRemotePitAndExecute( Version remoteVersion, long startTime ) { - String[] indices = request.getSearchRequest().indices(); + SearchRequest searchRequest = request.getSearchRequest(); + String[] indices = searchRequest.indices(); // Sends a REST request to the remote node to open a PIT - openPit(indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { + openPit(searchRequest, indices, pitKeepAlive(request), RejectAwareActionListener.wrap(pitId -> { ActionListener listenerWithClosePit = ActionListener.runAfter( listenerWithRelocations, () -> closePit(pitId, RejectAwareActionListener.wrap(v -> closeRestClientAndRun(restClient, () -> {}), e -> { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 58c7adf1f18a0..61386d54c70b8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -17,6 +17,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -28,6 +29,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RetryListener; import org.elasticsearch.rest.RestStatus; @@ -72,12 +74,17 @@ public static void lookupRemoteVersion(RejectAwareActionListener listen * @param client REST client for the remote cluster */ public static void openPit( + SearchRequest request, String[] indices, TimeValue keepAlive, RejectAwareActionListener listener, ThreadPool threadPool, RestClient client ) { + // The routing and preference parameters can be set on a PIT request. However, these are currently not used + // by either scroll nor PIT, so we assert here in case that changes + assert request.routing() == null : "Routing is set in the search request, but is not being used when opening the PIT."; + assert request.preference() == null : "Preference is set in the search request, but is not being used when opening the PIT."; execute(RemoteRequestBuilders.openPit(indices, keepAlive), OPEN_PIT_PARSER, listener, threadPool, client); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 547057c953518..12c2c9aa07521 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -502,7 +502,7 @@ public void testLocalReindexingRequestFailsToOpenPit() { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); - + fail("fix") final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -512,7 +512,10 @@ public void testLocalReindexingRequestFailsToOpenPit() { mock(ReindexSslConfig.class), null, mock(TransportService.class), - mock(ReindexRelocationNodePicker.class) + mock(ReindexRelocationNodePicker.class), + // TODO - Do these tests need updating to set it to true? + // Will default REINDEX_PIT_SEARCH_FEATURE to false + mock(FeatureService.class) ); final ReindexRequest request = new ReindexRequest(); @@ -576,7 +579,9 @@ public ExecutorService executor(String name) { mock(ReindexSslConfig.class), null, mock(TransportService.class), - mock(ReindexRelocationNodePicker.class) + mock(ReindexRelocationNodePicker.class), + // Will default REINDEX_PIT_SEARCH_FEATURE to false + mock(FeatureService.class) ); final ReindexRequest request = new ReindexRequest(); @@ -805,7 +810,9 @@ public ExecutorService executor(String name) { sslConfig, null, mock(TransportService.class), - mock(ReindexRelocationNodePicker.class) + mock(ReindexRelocationNodePicker.class), + // Will default REINDEX_PIT_SEARCH_FEATURE to false + mock(FeatureService.class) ); BulkByScrollTask task = new BulkByScrollTask( diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index a8920614143d3..0609cdc00895b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -18,6 +18,7 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; @@ -455,10 +456,13 @@ public void testOpenPitSuccess() { when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); mockSuccess(response); + String index = randomAlphaOfLength(between(1, 10)); + SearchRequest searchRequest = new SearchRequest().indices(index); AtomicBoolean success = new AtomicBoolean(false); BytesReference[] capturedPitId = new BytesReference[1]; RemoteReindexingUtils.openPit( - new String[] { randomAlphaOfLength(between(1, 10)) }, + searchRequest, + new String[] { index }, TimeValue.timeValueMillis(between(1, 60000)), RejectAwareActionListener.wrap(pitId -> { capturedPitId[0] = pitId; @@ -477,9 +481,12 @@ public void testOpenPitSuccess() { public void testOpenPitTooManyRequestsTriggersRejection() throws Exception { mockFailure(new ResponseException(rejectionResponse429())); + String index = randomAlphaOfLength(between(1, 10)); + SearchRequest searchRequest = new SearchRequest().indices(index); AtomicBoolean rejected = new AtomicBoolean(false); RemoteReindexingUtils.openPit( - new String[] { randomAlphaOfLength(between(1, 10)) }, + searchRequest, + new String[] { index }, randomPositiveTimeValue(), RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> fail("unexpected failure"), e -> rejected.set(true)), threadPool, @@ -503,9 +510,12 @@ public void testOpenPitHttpErrorTriggersFailure() throws Exception { when(response.getRequestLine()).thenReturn(requestLine); mockFailure(new ResponseException(response)); + String index = randomAlphaOfLength(between(1, 10)); + SearchRequest searchRequest = new SearchRequest().indices(index); AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.openPit( - new String[] { randomAlphaOfLength(between(1, 10)) }, + searchRequest, + new String[] { index }, randomPositiveTimeValue(), RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { assertTrue(e instanceof ElasticsearchStatusException); @@ -527,9 +537,12 @@ public void testOpenPitInvalidJsonTriggersFailure() { when(response.getEntity()).thenReturn(new StringEntity(invalidJson, ContentType.APPLICATION_JSON)); mockSuccess(response); + String index = randomAlphaOfLength(between(1, 10)); + SearchRequest searchRequest = new SearchRequest().indices(index); AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.openPit( - new String[] { randomAlphaOfLength(between(1, 10)) }, + searchRequest, + new String[] { index }, randomPositiveTimeValue(), RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { assertTrue(e instanceof ElasticsearchException); @@ -551,9 +564,12 @@ public void testOpenPitMissingIdFieldTriggersFailure() { when(response.getEntity()).thenReturn(new StringEntity(json, ContentType.APPLICATION_JSON)); mockSuccess(response); + String index = randomAlphaOfLength(between(1, 10)); + SearchRequest searchRequest = new SearchRequest().indices(index); AtomicBoolean failed = new AtomicBoolean(false); RemoteReindexingUtils.openPit( - new String[] { randomAlphaOfLength(between(1, 10)) }, + searchRequest, + new String[] { index }, randomPositiveTimeValue(), RejectAwareActionListener.wrap(v -> fail("unexpected success"), e -> { assertTrue(e instanceof ElasticsearchException); @@ -566,6 +582,56 @@ public void testOpenPitMissingIdFieldTriggersFailure() { assertTrue("onFailure should have been called", failed.get()); } + /** + * Verifies that openPit throws AssertionError when the SearchRequest has routing set, + * since routing is not yet propagated to the PIT open request. + */ + public void testOpenPitFailsWhenRoutingSet() { + SearchRequest searchRequest = new SearchRequest().indices("index"); + searchRequest.routing("some-routing"); + AssertionError e = expectThrows( + AssertionError.class, + () -> RemoteReindexingUtils.openPit( + searchRequest, + new String[] { "index" }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + err -> fail("unexpected failure"), + err -> fail("unexpected rejection") + ), + threadPool, + client + ) + ); + assertThat(e.getMessage(), containsString("Routing is set")); + } + + /** + * Verifies that openPit throws AssertionError when the SearchRequest has preference set, + * since preference is not yet propagated to the PIT open request. + */ + public void testOpenPitFailsWhenPreferenceSet() { + SearchRequest searchRequest = new SearchRequest().indices("index"); + searchRequest.preference("_local"); + AssertionError e = expectThrows( + AssertionError.class, + () -> RemoteReindexingUtils.openPit( + searchRequest, + new String[] { "index" }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + err -> fail("unexpected failure"), + err -> fail("unexpected rejection") + ), + threadPool, + client + ) + ); + assertThat(e.getMessage(), containsString("Preference is set")); + } + /** * Verifies that closePit invokes onResponse when the remote returns a successful close PIT response. */ From 4e80fbe7d93db7e30f23fff74c0e10b8e601af2f Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 11:54:49 +0000 Subject: [PATCH 37/45] Explicitly set allow_partial_search_results to false --- .../reindex/remote/RemoteReindexingUtils.java | 2 ++ .../reindex/remote/RemoteRequestBuilders.java | 2 ++ .../remote/RemoteReindexingUtilsTests.java | 25 +++++++++++++++++++ .../remote/RemoteRequestBuildersTests.java | 12 ++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 61386d54c70b8..5777fcbae4bd2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -85,6 +85,8 @@ public static void openPit( // by either scroll nor PIT, so we assert here in case that changes assert request.routing() == null : "Routing is set in the search request, but is not being used when opening the PIT."; assert request.preference() == null : "Preference is set in the search request, but is not being used when opening the PIT."; + assert request.allowPartialSearchResults() == false + : "allow_partial_search_results must be false when opening a PIT to match scroll search behavior"; execute(RemoteRequestBuilders.openPit(indices, keepAlive), OPEN_PIT_PARSER, listener, threadPool, client); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java index 7bccf652b2725..6f6789050286c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java @@ -182,12 +182,14 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query, return request; } + // TODO - Do we need to set the IndexFilter field here? https://github.com/elastic/elasticsearch-team/issues/2392 static Request openPit(String[] indices, TimeValue keepAlive) { StringBuilder path = new StringBuilder("/"); addIndices(path, indices); path.append("_pit"); Request request = new Request("POST", path.toString()); request.addParameter("keep_alive", keepAlive.getStringRep()); + request.addParameter("allow_partial_search_results", "false"); return request; } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 0609cdc00895b..9cad2c15a55c5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -632,6 +632,31 @@ public void testOpenPitFailsWhenPreferenceSet() { assertThat(e.getMessage(), containsString("Preference is set")); } + /** + * Verifies that openPit throws AssertionError when the SearchRequest has allowPartialSearchResults set to true + * since scroll search defaults to false + */ + public void testOpenPitFailsWhenAllowPartialSearchResultsTrue() { + SearchRequest searchRequest = new SearchRequest().indices("index"); + searchRequest.allowPartialSearchResults(true); + AssertionError e = expectThrows( + AssertionError.class, + () -> RemoteReindexingUtils.openPit( + searchRequest, + new String[] { "index" }, + randomPositiveTimeValue(), + RejectAwareActionListener.wrap( + v -> fail("unexpected success"), + err -> fail("unexpected failure"), + err -> fail("unexpected rejection") + ), + threadPool, + client + ) + ); + assertThat(e.getMessage(), containsString("allow_partial_search_results")); + } + /** * Verifies that closePit invokes onResponse when the remote returns a successful close PIT response. */ diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java index 2db74b2031c96..19e0e249e5dd6 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java @@ -346,6 +346,7 @@ public void testOpenPitSingleIndex() { assertEquals("POST", request.getMethod()); assertEquals("/" + index + "/_pit", request.getEndpoint()); assertThat(request.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + assertThat(request.getParameters(), hasEntry("allow_partial_search_results", "false")); } /** @@ -368,6 +369,7 @@ public void testOpenPitMultipleIndices() { assertEquals("POST", request.getMethod()); assertEquals(expectedPath.toString(), request.getEndpoint()); assertThat(request.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + assertThat(request.getParameters(), hasEntry("allow_partial_search_results", "false")); } /** @@ -379,11 +381,13 @@ public void testOpenPitNullOrEmptyIndices() { assertEquals("POST", nullRequest.getMethod()); assertEquals("/_pit", nullRequest.getEndpoint()); assertThat(nullRequest.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + assertThat(nullRequest.getParameters(), hasEntry("allow_partial_search_results", "false")); Request emptyRequest = openPit(new String[] {}, keepAlive); assertEquals("POST", emptyRequest.getMethod()); assertEquals("/_pit", emptyRequest.getEndpoint()); assertThat(emptyRequest.getParameters(), hasEntry("keep_alive", keepAlive.getStringRep())); + assertThat(emptyRequest.getParameters(), hasEntry("allow_partial_search_results", "false")); } /** @@ -395,6 +399,7 @@ public void testOpenPitEncodesSpecialCharactersInIndices() { Request request = openPit(new String[] { prefix1 + ",", prefix2 + "/" }, randomPositiveTimeValue()); assertEquals("POST", request.getMethod()); assertEquals("/" + prefix1 + "%2C," + prefix2 + "%2F/_pit", request.getEndpoint()); + assertThat(request.getParameters(), hasEntry("allow_partial_search_results", "false")); } /** @@ -403,10 +408,9 @@ public void testOpenPitEncodesSpecialCharactersInIndices() { public void testOpenPitKeepAliveParameter() { String index = randomAlphaOfLength(between(1, 10)); long millis = between(1, 100000); - assertThat( - openPit(new String[] { index }, timeValueMillis(millis)).getParameters(), - hasEntry("keep_alive", TimeValue.timeValueMillis(millis).getStringRep()) - ); + var params = openPit(new String[] { index }, timeValueMillis(millis)).getParameters(); + assertThat(params, hasEntry("allow_partial_search_results", "false")); + assertThat(params, hasEntry("keep_alive", TimeValue.timeValueMillis(millis).getStringRep())); int minutes = between(1, 60); assertThat( openPit(new String[] { index }, TimeValue.timeValueMinutes(minutes)).getParameters(), From 513040494e0806314789a5d381fc68f70a4fffed Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 12:30:06 +0000 Subject: [PATCH 38/45] Apply changes to local PIT request --- .../org/elasticsearch/reindex/Reindexer.java | 13 +- .../reindex/remote/RemoteReindexingUtils.java | 7 +- .../elasticsearch/reindex/ReindexerTests.java | 342 +++++++++++++++++- .../remote/RemoteReindexingUtilsTests.java | 2 +- 4 files changed, 350 insertions(+), 14 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 2a1a752c959a0..865f770b7729a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -105,7 +105,6 @@ import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.closePit; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.openPit; -import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE; public class Reindexer { @@ -269,8 +268,18 @@ private void openPitAndExecute( SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); + + // The routing and preference parameters can be set for a PIT request. However, scroll currently does not use these, + // so for parity we assert here in case that changes + assert searchRequest.routing() == null : "Routing is set in the search request, but is not being used when opening the PIT."; + assert searchRequest.preference() == null : "Preference is set in the search request, but is not being used when opening the PIT."; + assert searchRequest.allowPartialSearchResults() == null || searchRequest.allowPartialSearchResults() == false + : "allow_partial_search_results must be false when opening a PIT to match scroll search behavior"; + + // TODO - Do we need to set the IndexFilter field here? https://github.com/elastic/elasticsearch-team/issues/2392 OpenPointInTimeRequest pitRequest = new OpenPointInTimeRequest(indices).indicesOptions(searchRequest.indicesOptions()) - .keepAlive(pitKeepAlive(request)); + .keepAlive(pitKeepAlive(request)) + .allowPartialSearchResults(false); // NB this is a local request, so we call the TransportAction rather than issuing a REST call client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 5777fcbae4bd2..3c3bfba4ccecd 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.RejectAwareActionListener; import org.elasticsearch.index.reindex.RetryListener; import org.elasticsearch.rest.RestStatus; @@ -81,11 +80,11 @@ public static void openPit( ThreadPool threadPool, RestClient client ) { - // The routing and preference parameters can be set on a PIT request. However, these are currently not used - // by either scroll nor PIT, so we assert here in case that changes + // The routing and preference parameters can be set for a PIT request. However, scroll currently does not use these, + // so for parity we assert here in case that changes assert request.routing() == null : "Routing is set in the search request, but is not being used when opening the PIT."; assert request.preference() == null : "Preference is set in the search request, but is not being used when opening the PIT."; - assert request.allowPartialSearchResults() == false + assert request.allowPartialSearchResults() == null || request.allowPartialSearchResults() == false : "allow_partial_search_results must be false when opening a PIT to match scroll search behavior"; execute(RemoteRequestBuilders.openPit(indices, keepAlive), OPEN_PIT_PARSER, listener, threadPool, client); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 12c2c9aa07521..44e27a30f3452 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.ClosePointInTimeResponse; import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeResponse; import org.elasticsearch.action.search.SearchRequest; @@ -502,7 +503,7 @@ public void testLocalReindexingRequestFailsToOpenPit() { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); - fail("fix") + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -513,8 +514,6 @@ public void testLocalReindexingRequestFailsToOpenPit() { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - // TODO - Do these tests need updating to set it to true? - // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); @@ -580,7 +579,6 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); @@ -621,6 +619,288 @@ public ExecutorService executor(String name) { } } + /** + * Verifies that the OpenPointInTimeRequest built in openPitAndExecute has routing and preference unset, + * and allowPartialSearchResults explicitly set to false. + */ + public void testLocalOpenPitRequestHasExpectedProperties() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final OpenPitCapturingClient client = new OpenPitCapturingClient(getTestName()); + try { + final TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + mock(ReindexRelocationNodePicker.class), + mock(FeatureService.class) + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + final PlainActionFuture initFuture = new PlainActionFuture<>(); + reindexer.initTask(task, request, initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l))); + initFuture.actionGet(); + + OpenPointInTimeRequest pitRequest = client.getCapturedPitRequest(); + assertNotNull("Expected OpenPointInTimeRequest to have been captured", pitRequest); + assertNull("routing should not be set", pitRequest.routing()); + assertNull("preference should not be set", pitRequest.preference()); + assertFalse("allowPartialSearchResults should be false", pitRequest.allowPartialSearchResults()); + } finally { + terminate(threadPool); + } + } finally { + client.shutdown(); + } + } + + /** + * Verifies that openPitAndExecute throws AssertionError when the SearchRequest has routing set. + */ + public void testLocalOpenPitFailsWhenRoutingSet() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final OpenPitCapturingClient client = new OpenPitCapturingClient(getTestName()); + try { + final TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + mock(ReindexRelocationNodePicker.class), + mock(FeatureService.class) + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + request.getSearchRequest().routing("r1"); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + final PlainActionFuture initFuture = new PlainActionFuture<>(); + Throwable e = expectThrows( + Throwable.class, + () -> reindexer.initTask( + task, + request, + initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l)) + ) + ); + assertThat(ExceptionsHelper.unwrapCause(e).getMessage(), containsString("Routing is set in the search request")); + } finally { + terminate(threadPool); + } + } finally { + client.shutdown(); + } + } + + /** + * Verifies that openPitAndExecute throws AssertionError when the SearchRequest has preference set. + */ + public void testLocalOpenPitFailsWhenPreferenceSet() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final OpenPitCapturingClient client = new OpenPitCapturingClient(getTestName()); + try { + final TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + mock(ReindexRelocationNodePicker.class), + mock(FeatureService.class) + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + request.getSearchRequest().preference("_local"); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + final PlainActionFuture initFuture = new PlainActionFuture<>(); + Throwable e = expectThrows( + Throwable.class, + () -> reindexer.initTask( + task, + request, + initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l)) + ) + ); + assertThat(ExceptionsHelper.unwrapCause(e).getMessage(), containsString("Preference is set in the search request")); + } finally { + terminate(threadPool); + } + } finally { + client.shutdown(); + } + } + + /** + * Verifies that openPitAndExecute throws AssertionError when the SearchRequest has allowPartialSearchResults set to true. + */ + public void testLocalOpenPitFailsWhenAllowPartialSearchResultsTrue() { + assumeTrue("PIT search must be enabled", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + + final OpenPitCapturingClient client = new OpenPitCapturingClient(getTestName()); + try { + final TestThreadPool threadPool = new TestThreadPool(getTestName()) { + @Override + public ExecutorService executor(String name) { + return DIRECT_EXECUTOR_SERVICE; + } + }; + try { + final ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode localNode = DiscoveryNodeUtils.builder("local-node").build(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); + when(clusterService.localNode()).thenReturn(localNode); + + final ProjectResolver projectResolver = mock(ProjectResolver.class); + when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + + final Reindexer reindexer = new Reindexer( + clusterService, + projectResolver, + client, + threadPool, + mock(ScriptService.class), + mock(ReindexSslConfig.class), + null, + mock(TransportService.class), + mock(ReindexRelocationNodePicker.class), + mock(FeatureService.class) + ); + + final ReindexRequest request = new ReindexRequest(); + request.setSourceIndices("source"); + request.setDestIndex("dest"); + request.setSlices(1); + request.getSearchRequest().allowPartialSearchResults(true); + + final BulkByScrollTask task = new BulkByScrollTask( + randomLong(), + "reindex", + "reindex", + "test", + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + false + ); + + final PlainActionFuture initFuture = new PlainActionFuture<>(); + Throwable e = expectThrows( + Throwable.class, + () -> reindexer.initTask( + task, + request, + initFuture.delegateFailure((l, v) -> reindexer.execute(task, request, client, l)) + ) + ); + assertThat( + ExceptionsHelper.unwrapCause(e).getMessage(), + containsString("allow_partial_search_results must be false when opening a PIT") + ); + } finally { + terminate(threadPool); + } + } finally { + client.shutdown(); + } + } + /** * Client that succeeds on OpenPointInTime and Search (empty results) but fails on ClosePointInTime. * Used to verify that PIT close failures are logged but don't propagate to the main listener. @@ -652,6 +932,7 @@ protected void SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0) ); listener.onResponse((Response) response); + response.decRef(); return; } if (action == TransportClosePointInTimeAction.TYPE && request instanceof ClosePointInTimeRequest) { @@ -697,6 +978,56 @@ void shutdown() { } } + /** + * Client that captures the OpenPointInTimeRequest when received and returns success. + * Used to verify the local PIT request has the expected properties. + */ + private static final class OpenPitCapturingClient extends NoOpClient { + private final TestThreadPool threadPool; + private volatile OpenPointInTimeRequest capturedPitRequest; + + OpenPitCapturingClient(String threadPoolName) { + super(new TestThreadPool(threadPoolName), TestProjectResolvers.DEFAULT_PROJECT_ONLY); + this.threadPool = (TestThreadPool) super.threadPool(); + } + + OpenPointInTimeRequest getCapturedPitRequest() { + return capturedPitRequest; + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (action == TransportOpenPointInTimeAction.TYPE && request instanceof OpenPointInTimeRequest pitRequest) { + capturedPitRequest = pitRequest; + OpenPointInTimeResponse response = new OpenPointInTimeResponse(new BytesArray("pit-id"), 1, 1, 0, 0); + listener.onResponse((Response) response); + return; + } + if (action == TransportSearchAction.TYPE && request instanceof SearchRequest) { + SearchResponse response = SearchResponseUtils.successfulResponse( + SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0) + ); + listener.onResponse((Response) response); + response.decRef(); + return; + } + if (action == TransportClosePointInTimeAction.TYPE && request instanceof ClosePointInTimeRequest) { + listener.onResponse((Response) new ClosePointInTimeResponse(true, 1)); + return; + } + super.doExecute(action, request, listener); + } + + void shutdown() { + terminate(threadPool); + } + } + // --- helpers --- private static final String REMOTE_PIT_TEST_VERSION_JSON = "{\"version\":{\"number\":\"7.10.0\"},\"tagline\":\"You Know, for Search\"}"; @@ -811,7 +1142,6 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); @@ -891,7 +1221,6 @@ private static Reindexer reindexerWithRelocation(ClusterService clusterService, null, transportService, mock(ReindexRelocationNodePicker.class), - // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); } @@ -907,7 +1236,6 @@ private static Reindexer reindexerWithRelocationAndMetrics(final ReindexMetrics metrics, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 9cad2c15a55c5..8bc18bdb49778 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -18,10 +18,10 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; From 32ed73f63a52a1c1dfe1a6c26051b64356e8c20a Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 12:31:30 +0000 Subject: [PATCH 39/45] Minor tweaks --- .../reindex/remote/RemoteReindexingUtils.java | 10 +++++++--- .../reindex/remote/RemoteReindexingUtilsTests.java | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java index 3c3bfba4ccecd..70e6b22965caf 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteReindexingUtils.java @@ -127,9 +127,13 @@ public static void lookupRemoteVersionWithRetries( RestClient client, RejectAwareActionListener delegate ) { - RetryListener retryListener = new RetryListener<>(logger, threadPool, backoffPolicy, listener -> { - lookupRemoteVersion(listener, threadPool, client); - }, delegate); + RetryListener retryListener = new RetryListener<>( + logger, + threadPool, + backoffPolicy, + listener -> lookupRemoteVersion(listener, threadPool, client), + delegate + ); lookupRemoteVersion(retryListener, threadPool, client); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 8bc18bdb49778..9fce22651dac6 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -51,7 +51,6 @@ import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.wrapExceptionToPreserveStatus; import static org.hamcrest.Matchers.containsString; -import static org.junit.Assert.assertArrayEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -367,7 +366,7 @@ public void testLookupRemoteVersionWithRetriesSucceedsOnRetry() throws Exception /** * Verifies that lookupRemoteVersionWithRetries propagates failure when retries are exhausted. */ - public void testLookupRemoteVersionWithRetriesExhaustedPropagatesFailure() throws Exception { + public void testLookupRemoteVersionWithRetriesExhaustedPropagatesFailure() { Response rejectionResponse = rejectionResponse429(); doAnswer(inv -> { ((ResponseListener) inv.getArgument(1)).onFailure(new ResponseException(rejectionResponse)); From 6217e57486c33d2b129c569fa781a24e55bf4afb Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 12:52:52 +0000 Subject: [PATCH 40/45] Fix cluster feature merge error --- .../org/elasticsearch/reindex/Reindexer.java | 12 ++++-- .../elasticsearch/reindex/ReindexerTests.java | 37 +++++++++++++++---- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 865f770b7729a..41120e52f267e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -102,7 +102,7 @@ import static java.util.Collections.synchronizedList; import static org.elasticsearch.common.BackoffPolicy.exponentialBackoff; import static org.elasticsearch.index.VersionType.INTERNAL; -import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED; +import static org.elasticsearch.reindex.ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.closePit; import static org.elasticsearch.reindex.remote.RemoteReindexingUtils.openPit; @@ -167,11 +167,15 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl final boolean isRemote = request.getRemoteInfo() != null; Consumer workerAction = createWorkerAction(task, request, bulkClient, listenerWithRelocations, startTime, isRemote, false); - // Point-in-time searching is not enabled, so default to scroll - if (REINDEX_PIT_SEARCH_ENABLED == false) { + // Point-in-time searching is disabled, so default to scroll + if (featureService.clusterHasFeature(clusterService.state(), REINDEX_PIT_SEARCH_FEATURE) == false) { executePaginatedSearch(task, request, listenerWithRelocations, workerAction, null); } - // Point-in-time searching is enabled, and this is a remote request + /** + * Point-in-time searching is enabled + * As this is a request to reindex from remote, we need to determine the remote version prior to execution + * NB {@link ReindexRequest} forbids remote requests and slices > 1, so we're guaranteed to be running on the only slice + */ else if (isRemote) { lookupRemoteVersionAndExecute(task, request, bulkClient, listenerWithRelocations, workerAction, startTime); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java index 44e27a30f3452..7c58503f1d477 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexerTests.java @@ -504,6 +504,9 @@ public void testLocalReindexingRequestFailsToOpenPit() { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -514,7 +517,7 @@ public void testLocalReindexingRequestFailsToOpenPit() { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -569,6 +572,9 @@ public ExecutorService executor(String name) { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -579,7 +585,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -643,6 +649,9 @@ public ExecutorService executor(String name) { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -653,7 +662,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -711,6 +720,9 @@ public ExecutorService executor(String name) { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -721,7 +733,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -781,6 +793,9 @@ public ExecutorService executor(String name) { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -791,7 +806,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -851,6 +866,9 @@ public ExecutorService executor(String name) { final ProjectResolver projectResolver = mock(ProjectResolver.class); when(projectResolver.getProjectState(any())).thenReturn(ClusterState.EMPTY_STATE.projectState(Metadata.DEFAULT_PROJECT_ID)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + final Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -861,7 +879,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); final ReindexRequest request = new ReindexRequest(); @@ -1132,6 +1150,9 @@ public ExecutorService executor(String name) { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); ReindexSslConfig sslConfig = new ReindexSslConfig(environment.settings(), environment, mock(ResourceWatcherService.class)); + FeatureService featureService = mock(FeatureService.class); + when(featureService.clusterHasFeature(any(), eq(ReindexPlugin.REINDEX_PIT_SEARCH_FEATURE))).thenReturn(true); + Reindexer reindexer = new Reindexer( clusterService, projectResolver, @@ -1142,7 +1163,7 @@ public ExecutorService executor(String name) { null, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), - mock(FeatureService.class) + featureService ); BulkByScrollTask task = new BulkByScrollTask( @@ -1221,6 +1242,7 @@ private static Reindexer reindexerWithRelocation(ClusterService clusterService, null, transportService, mock(ReindexRelocationNodePicker.class), + // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); } @@ -1236,6 +1258,7 @@ private static Reindexer reindexerWithRelocationAndMetrics(final ReindexMetrics metrics, mock(TransportService.class), mock(ReindexRelocationNodePicker.class), + // Will default REINDEX_PIT_SEARCH_FEATURE to false mock(FeatureService.class) ); } From 2e181ac6a5477af4fd69b6b0f349c49010d52f11 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Thu, 5 Mar 2026 12:57:54 +0000 Subject: [PATCH 41/45] Fix test comments --- .../reindex/remote/RemoteReindexingUtilsTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java index 9fce22651dac6..c850090edfc05 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteReindexingUtilsTests.java @@ -582,8 +582,7 @@ public void testOpenPitMissingIdFieldTriggersFailure() { } /** - * Verifies that openPit throws AssertionError when the SearchRequest has routing set, - * since routing is not yet propagated to the PIT open request. + * Verifies that openPit throws AssertionError when the SearchRequest has routing set */ public void testOpenPitFailsWhenRoutingSet() { SearchRequest searchRequest = new SearchRequest().indices("index"); @@ -607,8 +606,7 @@ public void testOpenPitFailsWhenRoutingSet() { } /** - * Verifies that openPit throws AssertionError when the SearchRequest has preference set, - * since preference is not yet propagated to the PIT open request. + * Verifies that openPit throws AssertionError when the SearchRequest has preference set */ public void testOpenPitFailsWhenPreferenceSet() { SearchRequest searchRequest = new SearchRequest().indices("index"); From e2b2fb8cbd9e56733d10b922289f54456c2622da Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Tue, 10 Mar 2026 10:58:53 +0000 Subject: [PATCH 42/45] Tidy up merge --- .../org/elasticsearch/reindex/Reindexer.java | 76 ++++--------------- 1 file changed, 16 insertions(+), 60 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 3d14609167d71..cae082e8ddaac 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -171,11 +171,11 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl ); final boolean isRemote = request.getRemoteInfo() != null; - Consumer workerAction = createWorkerAction(task, request, bulkClient, listenerWithRelocations, startTime, isRemote, false); + Consumer workerAction = createWorkerAction(task, request, bulkClient, responseListener); // Point-in-time searching is disabled, so default to scroll if (featureService.clusterHasFeature(clusterService.state(), REINDEX_PIT_SEARCH_FEATURE) == false) { - executePaginatedSearch(task, request, listenerWithRelocations, workerAction, null); + executePaginatedSearch(task, request, responseListener, workerAction, null); } /** * Point-in-time searching is enabled @@ -183,11 +183,11 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl * NB {@link ReindexRequest} forbids remote requests and slices > 1, so we're guaranteed to be running on the only slice */ else if (isRemote) { - lookupRemoteVersionAndExecute(task, request, bulkClient, listenerWithRelocations, workerAction, startTime); + lookupRemoteVersionAndExecute(task, request, bulkClient, responseListener, workerAction); } // Point-in-time searching is enabled, and this is a local request else { - openPitAndExecute(task, request, bulkClient, listenerWithRelocations, startTime); + openPitAndExecute(task, request, bulkClient, responseListener); } } @@ -199,16 +199,8 @@ private Consumer createWorkerAction( BulkByScrollTask task, ReindexRequest request, Client bulkClient, - ActionListener listener, - long startTime, - boolean isRemote, - boolean listenerAlreadyHasMetrics + ActionListener listener ) { - // PIT paths wrap the listener with metrics before executing PIT specific REST calls. - // When scroll is then used, we avoid double-recording by skipping the wrapper here. - ActionListener workerListener = listenerAlreadyHasMetrics - ? listener - : workerListenerWithRelocationAndMetrics(listener, startTime, isRemote); return remoteVersion -> { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); ParentTaskAssigningClient assigningBulkClient = new ParentTaskAssigningClient(bulkClient, clusterService.localNode(), task); @@ -222,7 +214,7 @@ private Consumer createWorkerAction( projectResolver.getProjectState(clusterService.state()), reindexSslConfig, request, - responseListener, + listener, remoteVersion ); searchAction.start(); @@ -266,16 +258,8 @@ private void openPitAndExecute( BulkByScrollTask task, ReindexRequest request, Client bulkClient, - ActionListener listenerWithRelocations, - long startTime + ActionListener listener ) { - // Wrap with metrics so failures before the worker runs (PIT open) are recorded - final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( - listenerWithRelocations, - startTime, - false - ); - SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); @@ -292,7 +276,7 @@ private void openPitAndExecute( .allowPartialSearchResults(false); // NB this is a local request, so we call the TransportAction rather than issuing a REST call - client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listenerWithMetrics.delegateFailureAndWrap((l, pitResponse) -> { + client.execute(TransportOpenPointInTimeAction.TYPE, pitRequest, listener.delegateFailureAndWrap((l, pitResponse) -> { BytesReference pitId = pitResponse.getPointInTimeId(); ActionListener listenerWithClosePit = ActionListener.runAfter( l, @@ -302,15 +286,7 @@ private void openPitAndExecute( ActionListener.wrap(r -> {}, e -> logger.warn("Failed to close local PIT", e)) ) ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - false, - true - ); + Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit); // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, null); })); @@ -327,16 +303,8 @@ private void lookupRemoteVersionAndExecute( ReindexRequest request, Client bulkClient, ActionListener listener, - Consumer workerAction, - long startTime + Consumer workerAction ) { - // Wrap with metrics so failures before the worker runs (version lookup, PIT open) are recorded -// final ActionListener listenerWithMetrics = workerListenerWithRelocationAndMetrics( -// listenerWithRelocations, -// startTime, -// true -// ); - RemoteInfo remoteInfo = request.getRemoteInfo(); assert reindexSslConfig != null : "Reindex ssl config must be set"; RestClient restClient = buildRestClient(remoteInfo, reindexSslConfig, task.getId(), synchronizedList(new ArrayList<>())); @@ -345,25 +313,22 @@ private void lookupRemoteVersionAndExecute( public void onResponse(Version version) { boolean canUsePit = version.onOrAfter(Version.V_7_10_0); if (canUsePit) { - openRemotePitAndExecute(task, request, bulkClient, listenerWithMetrics, restClient, version, startTime); + openRemotePitAndExecute(task, request, bulkClient, listener, restClient, version); } // Default to scroll-based search else { - closeRestClientAndRun( - restClient, - () -> executePaginatedSearch(task, request, listenerWithRelocations, workerAction, version) - ); + closeRestClientAndRun(restClient, () -> executePaginatedSearch(task, request, listener, workerAction, version)); } } @Override public void onFailure(Exception e) { - closeRestClientAndRun(restClient, () -> listenerWithMetrics.onFailure(e)); + closeRestClientAndRun(restClient, () -> listener.onFailure(e)); } @Override public void onRejection(Exception e) { - closeRestClientAndRun(restClient, () -> listenerWithMetrics.onFailure(e)); + closeRestClientAndRun(restClient, () -> listener.onFailure(e)); } }; @@ -388,8 +353,7 @@ private void openRemotePitAndExecute( Client bulkClient, ActionListener listenerWithRelocations, RestClient restClient, - Version remoteVersion, - long startTime + Version remoteVersion ) { SearchRequest searchRequest = request.getSearchRequest(); String[] indices = searchRequest.indices(); @@ -405,15 +369,7 @@ private void openRemotePitAndExecute( closeRestClientAndRun(restClient, () -> {}); }), threadPool, restClient) ); - Consumer workerActionWithClosePit = createWorkerAction( - task, - request, - bulkClient, - listenerWithClosePit, - startTime, - true, - true - ); + Consumer workerActionWithClosePit = createWorkerAction(task, request, bulkClient, listenerWithClosePit); // TODO - Pass the point-in-time ID into the BulkByPaginatedSearchParallelizationHelper to be used executePaginatedSearch(task, request, listenerWithClosePit, workerActionWithClosePit, remoteVersion); }, From 724866cbe1eac96529ea7e2a0e6d398688b76b69 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Tue, 10 Mar 2026 17:55:37 +0000 Subject: [PATCH 43/45] Fix failing integ tests --- .../xpack/core/security/user/InternalUsers.java | 5 +++++ .../xpack/core/security/user/InternalUsersTests.java | 4 ++++ .../org/elasticsearch/upgrades/DataStreamsUpgradeIT.java | 2 ++ 3 files changed, 11 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index 24cfdcc143984..5883992dfe349 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -26,6 +26,8 @@ import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.downsample.DownsampleAction; import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.index.reindex.ReindexAction; @@ -217,6 +219,7 @@ public class InternalUsers { RoleDescriptor.IndicesPrivileges.builder() .indices("*") .privileges( + "read", GetDataStreamAction.NAME, RolloverAction.NAME, IndicesStatsAction.NAME, @@ -233,6 +236,8 @@ public class InternalUsers { TransportUpdateSettingsAction.TYPE.name(), RefreshAction.NAME, ReindexAction.NAME, + TransportClosePointInTimeAction.TYPE.name(), + TransportOpenPointInTimeAction.TYPE.name(), TransportSearchAction.NAME, TransportBulkAction.NAME, TransportIndexAction.NAME, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java index 6cdd00d2e98f5..c74cdfb18ce6a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java @@ -31,6 +31,8 @@ import org.elasticsearch.action.downsample.DownsampleAction; import org.elasticsearch.action.get.TransportGetAction; import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.cluster.metadata.DataStream; @@ -348,6 +350,8 @@ public void testReindexDataStreamUser() { TransportUpdateSettingsAction.TYPE.name(), RefreshAction.NAME, ReindexAction.NAME, + TransportClosePointInTimeAction.TYPE.name(), + TransportOpenPointInTimeAction.TYPE.name(), TransportSearchAction.NAME, TransportBulkAction.NAME, TransportIndexAction.NAME, diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index 40fc248fc9deb..d92d596ab42d4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Strings; +import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.xcontent.json.JsonXContent; import org.hamcrest.Matchers; @@ -189,6 +190,7 @@ public void testDataStreamValidationDoesNotBreakUpgrade() throws Exception { } public void testUpgradeDataStream() throws Exception { + assumeFalse("PIT search cannot be used on closed indices", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); /* * This test tests upgrading a "normal" data stream (dataStreamName), and upgrading a data stream that was originally just an * ordinary index that was converted to a data stream (dataStreamFromNonDataStreamIndices). From 730086ddc7399f7751158e2d58117c371829c3d2 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Tue, 10 Mar 2026 17:56:22 +0000 Subject: [PATCH 44/45] Add TODO --- .../java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index d92d596ab42d4..4a3e23a148c3a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -190,6 +190,7 @@ public void testDataStreamValidationDoesNotBreakUpgrade() throws Exception { } public void testUpgradeDataStream() throws Exception { + // TODO - https://github.com/elastic/elasticsearch-team/issues/2410 assumeFalse("PIT search cannot be used on closed indices", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); /* * This test tests upgrading a "normal" data stream (dataStreamName), and upgrading a data stream that was originally just an From 57567604726e41b88222bfda9e0d2ba6bcb57e45 Mon Sep 17 00:00:00 2001 From: Joshua Adams Date: Wed, 11 Mar 2026 10:35:38 +0000 Subject: [PATCH 45/45] Add feature flag block to testCancelEndpointEndToEndSynchronously and testCancelEndpointEndToEndAsynchronously --- .../org/elasticsearch/reindex/management/ReindexCancelIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/reindex-management/src/internalClusterTest/java/org/elasticsearch/reindex/management/ReindexCancelIT.java b/modules/reindex-management/src/internalClusterTest/java/org/elasticsearch/reindex/management/ReindexCancelIT.java index 58ae056ecf736..f6f0b1adb3fa8 100644 --- a/modules/reindex-management/src/internalClusterTest/java/org/elasticsearch/reindex/management/ReindexCancelIT.java +++ b/modules/reindex-management/src/internalClusterTest/java/org/elasticsearch/reindex/management/ReindexCancelIT.java @@ -91,6 +91,8 @@ public void setup() { * We test synchronous (?wait_for_completion=true) invocation of the _cancel endpoint in this test. */ public void testCancelEndpointEndToEndSynchronously() throws Exception { + assumeFalse("scroll-based reindex uses a different code path", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + final TaskId parentTaskId = startAsyncThrottledReindex(); final TaskInfo running = getRunningTask(parentTaskId); @@ -139,6 +141,8 @@ public void testCancelEndpointEndToEndSynchronously() throws Exception { /** Same test as above but calling _cancel asynchronously and wrapping assertions after cancellation in assertBusy. */ public void testCancelEndpointEndToEndAsynchronously() throws Exception { + assumeFalse("scroll-based reindex uses a different code path", ReindexPlugin.REINDEX_PIT_SEARCH_ENABLED); + final TaskId parentTaskId = startAsyncThrottledReindex(); final TaskInfo running = getRunningTask(parentTaskId);