From 3089fd28c5af8c2c0a5457fb5e36998c0f3e64a4 Mon Sep 17 00:00:00 2001
From: Gerard Soldevila <gerard.soldevila@elastic.co>
Date: Tue, 16 Dec 2025 13:05:26 +0100
Subject: [PATCH] Retry on "all shards failed" from ES

---
 .../catch_retryable_es_client_errors.test.ts  | 30 ++++++++++++++++++-
 .../catch_retryable_es_client_errors.ts       |  9 +++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts
index f76ac3f85cb90..5621a94e059bb 100644
--- a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts
+++ b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts
@@ -86,7 +86,7 @@ describe('catchRetryableEsClientErrors', () => {
 });
 
 describe('catchRetryableSearchPhaseExecutionException', () => {
-  it('retries search phase execution exception ', async () => {
+  it('retries search phase execution exception', async () => {
     const error = new esErrors.ResponseError(
       elasticsearchClientMock.createApiResponse({
         body: {
@@ -115,6 +115,34 @@ describe('catchRetryableSearchPhaseExecutionException', () => {
       }
     `);
   });
+  it('retries search phase execution exception for "all shards failed"', async () => {
+    const error = new esErrors.ResponseError(
+      elasticsearchClientMock.createApiResponse({
+        body: {
+          error: {
+            type: 'search_phase_execution_exception',
+            caused_by: {
+              type: 'search_phase_execution_exception',
+              reason: 'all shards failed',
+            },
+          },
+        },
+      })
+    );
+    expect(
+      ((await Promise.reject(error).catch(catchRetryableSearchPhaseExecutionException)) as any).left
+    ).toMatchInlineSnapshot(`
+      Object {
+        "error": [ResponseError: search_phase_execution_exception
+      	Caused by:
+      		search_phase_execution_exception: all shards failed],
+        "message": "search_phase_execution_exception
+      	Caused by:
+      		search_phase_execution_exception: all shards failed",
+        "type": "retryable_es_client_error",
+      }
+    `);
+  });
   it('does not retry other errors', async () => {
     const error = new esErrors.ResponseError(
       elasticsearchClientMock.createApiResponse({
diff --git a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts
index c19dca2f16eeb..ae991b511d90c 100644
--- a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts
+++ b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts
@@ -30,6 +30,11 @@ const retryResponseStatuses = [
   504, // GatewayTimeout
 ];
 
+const temporarySearchPhaseExecutionExceptionReasons = [
+  'Search rejected due to missing shards',
+  'all shards failed',
+];
+
 export const catchRetryableEsClientErrors = (
   e: EsErrors.ElasticsearchClientError
 ): Either.Either<RetryableEsClientError, never> => {
@@ -49,7 +54,9 @@ export const catchRetryableSearchPhaseExecutionException = (
 ): Either.Either<RetryableEsClientError, never> => {
   if (
     e?.body?.error?.type === 'search_phase_execution_exception' &&
-    e?.body?.error?.caused_by?.reason?.includes('Search rejected due to missing shards')
+    temporarySearchPhaseExecutionExceptionReasons.some((reason) =>
+      e?.body?.error?.caused_by?.reason?.includes(reason)
+    )
   ) {
     return Either.left({
       type: 'retryable_es_client_error' as const,