diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c26c2845ec7a9..e1bd20ddcea99 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -166,6 +166,7 @@ src/core/packages/elasticsearch/client-server-mocks @elastic/kibana-core src/core/packages/elasticsearch/server @elastic/kibana-core src/core/packages/elasticsearch/server-internal @elastic/kibana-core src/core/packages/elasticsearch/server-mocks @elastic/kibana-core +src/core/packages/elasticsearch/server-utils @elastic/kibana-core src/core/packages/environment/server-internal @elastic/kibana-core src/core/packages/environment/server-mocks @elastic/kibana-core src/core/packages/execution-context/browser @elastic/kibana-core diff --git a/package.json b/package.json index b0f40f730591f..1fae843ec6974 100644 --- a/package.json +++ b/package.json @@ -322,6 +322,7 @@ "@kbn/core-elasticsearch-client-server-internal": "link:src/core/packages/elasticsearch/client-server-internal", "@kbn/core-elasticsearch-server": "link:src/core/packages/elasticsearch/server", "@kbn/core-elasticsearch-server-internal": "link:src/core/packages/elasticsearch/server-internal", + "@kbn/core-elasticsearch-server-utils": "link:src/core/packages/elasticsearch/server-utils", "@kbn/core-environment-server-internal": "link:src/core/packages/environment/server-internal", "@kbn/core-execution-context-browser": "link:src/core/packages/execution-context/browser", "@kbn/core-execution-context-browser-internal": "link:src/core/packages/execution-context/browser-internal", diff --git a/src/core/packages/elasticsearch/server-internal/index.ts b/src/core/packages/elasticsearch/server-internal/index.ts index c08a8affea1f7..e3a502633396f 100644 --- a/src/core/packages/elasticsearch/server-internal/index.ts +++ b/src/core/packages/elasticsearch/server-internal/index.ts @@ -31,5 +31,4 @@ export { CoreElasticsearchRouteHandlerContext } from './src/elasticsearch_route_ export { retryCallCluster, migrationRetryCallCluster } from './src/retry_call_cluster'; export { isInlineScriptingEnabled } from './src/is_scripting_enabled'; export { getCapabilitiesFromClient } from './src/get_capabilities'; -export { isRetryableEsClientError } from './src/retryable_es_client_errors'; export type { ClusterInfo } from './src/get_cluster_info'; diff --git a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.ts b/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.ts index 4289ed415e122..5914aa4a1b767 100644 --- a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.ts +++ b/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.ts @@ -7,8 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { isRetryableEsClientErrorMock } from './is_scripting_enabled.test.mocks'; import type { estypes } from '@elastic/elasticsearch'; +import { errors as esErrors } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { isInlineScriptingEnabled } from './is_scripting_enabled'; @@ -98,10 +98,6 @@ describe('isInlineScriptingEnabled', () => { }); describe('resiliency', () => { - beforeEach(() => { - isRetryableEsClientErrorMock.mockReset(); - }); - const mockSuccessOnce = () => { client.cluster.getSettings.mockResolvedValueOnce({ transient: {}, @@ -109,16 +105,22 @@ describe('isInlineScriptingEnabled', () => { defaults: {}, }); }; - const mockErrorOnce = () => { - client.cluster.getSettings.mockResponseImplementationOnce(() => { - throw Error('ERR CON REFUSED'); - }); + + const mockRetryableErrorOnce = () => { + client.cluster.getSettings.mockRejectedValueOnce( + new esErrors.ConnectionError( + 'Connection failed', + elasticsearchClientMock.createApiResponse() + ) + ); }; - it('retries the ES api call in case of retryable error', async () => { - isRetryableEsClientErrorMock.mockReturnValue(true); + const mockNonRetryableErrorOnce = () => { + client.cluster.getSettings.mockRejectedValueOnce(new Error('Non-retryable error')); + }; - mockErrorOnce(); + it('retries the ES api call in case of retryable error', async () => { + mockRetryableErrorOnce(); mockSuccessOnce(); await expect(isInlineScriptingEnabled({ client, maxRetryDelay: 1 })).resolves.toEqual(true); @@ -126,27 +128,23 @@ describe('isInlineScriptingEnabled', () => { }); it('throws in case of non-retryable error', async () => { - isRetryableEsClientErrorMock.mockReturnValue(false); - - mockErrorOnce(); + mockNonRetryableErrorOnce(); mockSuccessOnce(); await expect(isInlineScriptingEnabled({ client, maxRetryDelay: 0.1 })).rejects.toThrowError( - 'ERR CON REFUSED' + 'Non-retryable error' ); }); it('retries up to `maxRetries` times', async () => { - isRetryableEsClientErrorMock.mockReturnValue(true); - - mockErrorOnce(); - mockErrorOnce(); - mockErrorOnce(); + mockRetryableErrorOnce(); + mockRetryableErrorOnce(); + mockRetryableErrorOnce(); mockSuccessOnce(); await expect( isInlineScriptingEnabled({ client, maxRetryDelay: 0.1, maxRetries: 2 }) - ).rejects.toThrowError('ERR CON REFUSED'); + ).rejects.toThrowError('Connection failed'); expect(client.cluster.getSettings).toHaveBeenCalledTimes(3); }); }); diff --git a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.ts b/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.ts index a232eca821879..ffe25ec36eb7c 100644 --- a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.ts +++ b/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.ts @@ -8,8 +8,8 @@ */ import { defer, map, retry, timer, firstValueFrom, throwError } from 'rxjs'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { isRetryableEsClientError } from './retryable_es_client_errors'; const scriptAllowedTypesKey = 'script.allowed_types'; diff --git a/src/core/packages/elasticsearch/server-internal/tsconfig.json b/src/core/packages/elasticsearch/server-internal/tsconfig.json index cc9debba44954..e7f8c4795bc31 100644 --- a/src/core/packages/elasticsearch/server-internal/tsconfig.json +++ b/src/core/packages/elasticsearch/server-internal/tsconfig.json @@ -24,6 +24,7 @@ "@kbn/core-http-server-internal", "@kbn/core-execution-context-server-internal", "@kbn/core-elasticsearch-server", + "@kbn/core-elasticsearch-server-utils", "@kbn/core-elasticsearch-client-server-internal", "@kbn/core-test-helpers-deprecations-getters", "@kbn/config", diff --git a/src/core/packages/elasticsearch/server-utils/README.md b/src/core/packages/elasticsearch/server-utils/README.md new file mode 100644 index 0000000000000..ee6f1e146c2e0 --- /dev/null +++ b/src/core/packages/elasticsearch/server-utils/README.md @@ -0,0 +1,3 @@ +# @kbn/core-elasticsearch-server-utils + +Utilities for working with Elasticsearch diff --git a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.mocks.ts b/src/core/packages/elasticsearch/server-utils/index.ts similarity index 71% rename from src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.mocks.ts rename to src/core/packages/elasticsearch/server-utils/index.ts index 84f156c650fc8..615115176df56 100644 --- a/src/core/packages/elasticsearch/server-internal/src/is_scripting_enabled.test.mocks.ts +++ b/src/core/packages/elasticsearch/server-utils/index.ts @@ -7,10 +7,4 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export const isRetryableEsClientErrorMock = jest.fn(); - -jest.doMock('./retryable_es_client_errors', () => { - return { - isRetryableEsClientError: isRetryableEsClientErrorMock, - }; -}); +export { isRetryableEsClientError } from './src/is_retryable_es_client_error'; diff --git a/src/core/packages/elasticsearch/server-utils/jest.config.js b/src/core/packages/elasticsearch/server-utils/jest.config.js new file mode 100644 index 0000000000000..2c386b79c26a5 --- /dev/null +++ b/src/core/packages/elasticsearch/server-utils/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../../..', + roots: ['/src/core/packages/elasticsearch/server-utils'], +}; diff --git a/src/core/packages/elasticsearch/server-utils/kibana.jsonc b/src/core/packages/elasticsearch/server-utils/kibana.jsonc new file mode 100644 index 0000000000000..5adb27816aeb6 --- /dev/null +++ b/src/core/packages/elasticsearch/server-utils/kibana.jsonc @@ -0,0 +1,7 @@ +{ + "type": "shared-common", + "id": "@kbn/core-elasticsearch-server-utils", + "owner": "@elastic/kibana-core", + "group": "platform", + "visibility": "shared" +} diff --git a/src/core/packages/elasticsearch/server-utils/package.json b/src/core/packages/elasticsearch/server-utils/package.json new file mode 100644 index 0000000000000..a750142ec0711 --- /dev/null +++ b/src/core/packages/elasticsearch/server-utils/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/core-elasticsearch-server-utils", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} \ No newline at end of file diff --git a/src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.test.ts b/src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.test.ts similarity index 67% rename from src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.test.ts rename to src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.test.ts index 4851394f30d4a..71f998e29f7d7 100644 --- a/src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.test.ts +++ b/src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.test.ts @@ -9,7 +9,7 @@ import { errors as esErrors } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import { isRetryableEsClientError } from './retryable_es_client_errors'; +import { isRetryableEsClientError } from './is_retryable_es_client_error'; describe('isRetryableEsClientError', () => { describe('returns `false` for', () => { @@ -51,27 +51,34 @@ describe('isRetryableEsClientError', () => { expect(isRetryableEsClientError(error)).toEqual(true); }); - it('ResponseError of type snapshot_in_progress_exception', () => { + it.each([503, 504, 408, 410, 429])('ResponseError with %p status code', (statusCode) => { const error = new esErrors.ResponseError( elasticsearchClientMock.createApiResponse({ - body: { error: { type: 'snapshot_in_progress_exception' } }, + statusCode, + body: { error: { type: 'reason' } }, }) ); + expect(isRetryableEsClientError(error)).toEqual(true); }); - it.each([503, 504, 401, 403, 408, 410, 429])( - 'ResponseError with %p status code', - (statusCode) => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode, - body: { error: { type: 'reason' } }, - }) - ); + it('custom response status codes', () => { + const retryableError = new esErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 418, // I'm a retryable teapot + body: { error: { type: 'reason' } }, + }) + ); - expect(isRetryableEsClientError(error)).toEqual(true); - } - ); + const nonRetryableError = new esErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 503, // 503 is retryable by default but not in our custom retry codes + body: { error: { type: 'reason' } }, + }) + ); + + expect(isRetryableEsClientError(retryableError, [418])).toEqual(true); + expect(isRetryableEsClientError(nonRetryableError, [418])).toEqual(false); + }); }); }); diff --git a/src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.ts b/src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.ts similarity index 56% rename from src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.ts rename to src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.ts index 004c5f3b3a920..2fba983270706 100644 --- a/src/core/packages/elasticsearch/server-internal/src/retryable_es_client_errors.ts +++ b/src/core/packages/elasticsearch/server-utils/src/is_retryable_es_client_error.ts @@ -9,9 +9,7 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; -const retryResponseStatuses = [ - 401, // AuthorizationException - 403, // AuthenticationException +const DEFAULT_RETRY_STATUS_CODES = [ 408, // RequestTimeout 410, // Gone 429, // TooManyRequests -> ES circuit breaker @@ -21,20 +19,32 @@ const retryResponseStatuses = [ /** * Returns true if the given elasticsearch error should be retried - * by retry-based resiliency systems such as the SO migration, false otherwise. + * + * Retryable errors include: + * - NoLivingConnectionsError + * - ConnectionError + * - TimeoutError + * - ResponseError with status codes: + * - 408 RequestTimeout + * - 410 Gone + * - 429 TooManyRequests (ES circuit breaker) + * - 503 ServiceUnavailable + * - 504 GatewayTimeout + * - OR custom status codes if provided + * @param e The error to check + * @param customRetryStatusCodes Custom response status codes to consider as retryable + * @returns true if the error is retryable, false otherwise */ -export const isRetryableEsClientError = (e: EsErrors.ElasticsearchClientError): boolean => { +export const isRetryableEsClientError = ( + e: EsErrors.ElasticsearchClientError, + customRetryStatusCodes?: number[] +): boolean => { if ( e instanceof EsErrors.NoLivingConnectionsError || e instanceof EsErrors.ConnectionError || e instanceof EsErrors.TimeoutError || (e instanceof EsErrors.ResponseError && - (retryResponseStatuses.includes(e?.statusCode!) || - // ES returns a 400 Bad Request when trying to close or delete an - // index while snapshots are in progress. This should have been a 503 - // so once https://github.com/elastic/elasticsearch/issues/65883 is - // fixed we can remove this. - e?.body?.error?.type === 'snapshot_in_progress_exception')) + (customRetryStatusCodes ?? DEFAULT_RETRY_STATUS_CODES).includes(e?.statusCode!)) ) { return true; } diff --git a/src/core/packages/elasticsearch/server-utils/tsconfig.json b/src/core/packages/elasticsearch/server-utils/tsconfig.json new file mode 100644 index 0000000000000..c0417d524fdab --- /dev/null +++ b/src/core/packages/elasticsearch/server-utils/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts" + ], + "kbn_references": [ + "@kbn/core-elasticsearch-client-server-mocks" + ], + "exclude": [ + "target/**/*", + ] +} diff --git a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts index ad6dfbcdc2170..55d8f2807e649 100644 --- a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts +++ b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts @@ -62,19 +62,6 @@ describe('catchRetryableEsClientErrors', () => { type: 'retryable_es_client_error', }); }); - it('ResponseError of type snapshot_in_progress_exception', async () => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - body: { error: { type: 'snapshot_in_progress_exception' } }, - }) - ); - expect( - ((await Promise.reject(error).catch(catchRetryableEsClientErrors)) as any).left - ).toMatchObject({ - message: 'snapshot_in_progress_exception', - type: 'retryable_es_client_error', - }); - }); it.each([503, 401, 403, 408, 410, 429])( 'ResponseError with retryable status code (%d)', async (status) => { diff --git a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts index ff970b955873f..4e8b5be69b6de 100644 --- a/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts +++ b/src/core/packages/saved-objects/migration-server-internal/src/actions/catch_retryable_es_client_errors.ts @@ -9,7 +9,7 @@ import * as Either from 'fp-ts/Either'; import type { errors as EsErrors } from '@elastic/elasticsearch'; -import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-internal'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; export interface RetryableEsClientError { type: 'retryable_es_client_error'; @@ -17,10 +17,22 @@ export interface RetryableEsClientError { error?: Error; } +// Migrations also retry on Auth exceptions as this is a common failure for newly created +// clusters that might have misconfigured credentials. +const retryResponseStatuses = [ + 401, // AuthorizationException + 403, // AuthenticationException + 408, // RequestTimeout + 410, // Gone + 429, // TooManyRequests -> ES circuit breaker + 503, // ServiceUnavailable + 504, // GatewayTimeout +]; + export const catchRetryableEsClientErrors = ( e: EsErrors.ElasticsearchClientError ): Either.Either => { - if (isRetryableEsClientError(e)) { + if (isRetryableEsClientError(e, retryResponseStatuses)) { return Either.left({ type: 'retryable_es_client_error' as const, message: e?.message, diff --git a/src/core/packages/saved-objects/migration-server-internal/tsconfig.json b/src/core/packages/saved-objects/migration-server-internal/tsconfig.json index 31f3da33386f4..d73f6603c2d10 100644 --- a/src/core/packages/saved-objects/migration-server-internal/tsconfig.json +++ b/src/core/packages/saved-objects/migration-server-internal/tsconfig.json @@ -16,6 +16,7 @@ "@kbn/std", "@kbn/core-doc-links-server", "@kbn/core-elasticsearch-server", + "@kbn/core-elasticsearch-server-utils", "@kbn/core-elasticsearch-client-server-internal", "@kbn/core-saved-objects-common", "@kbn/core-saved-objects-server", diff --git a/tsconfig.base.json b/tsconfig.base.json index c606ad0cbbe19..f42f1caf8fe3d 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -412,6 +412,8 @@ "@kbn/core-elasticsearch-server-internal/*": ["src/core/packages/elasticsearch/server-internal/*"], "@kbn/core-elasticsearch-server-mocks": ["src/core/packages/elasticsearch/server-mocks"], "@kbn/core-elasticsearch-server-mocks/*": ["src/core/packages/elasticsearch/server-mocks/*"], + "@kbn/core-elasticsearch-server-utils": ["src/core/packages/elasticsearch/server-utils"], + "@kbn/core-elasticsearch-server-utils/*": ["src/core/packages/elasticsearch/server-utils/*"], "@kbn/core-environment-server-internal": ["src/core/packages/environment/server-internal"], "@kbn/core-environment-server-internal/*": ["src/core/packages/environment/server-internal/*"], "@kbn/core-environment-server-mocks": ["src/core/packages/environment/server-mocks"], diff --git a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts index 519ac21d3af30..adfd3eab0bbfb 100644 --- a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts +++ b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts @@ -6,22 +6,10 @@ */ import type { Logger } from '@kbn/core/server'; -import { errors as EsErrors } from '@elastic/elasticsearch'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 3; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: Error) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const retryTransientEsErrors = async ( @@ -37,7 +25,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... diff --git a/x-pack/platform/plugins/shared/alerting/tsconfig.json b/x-pack/platform/plugins/shared/alerting/tsconfig.json index 08010bc1ae93e..3e1af55718fe5 100644 --- a/x-pack/platform/plugins/shared/alerting/tsconfig.json +++ b/x-pack/platform/plugins/shared/alerting/tsconfig.json @@ -78,6 +78,7 @@ "@kbn/elastic-assistant-common", "@kbn/response-ops-recurring-schedule-form", "@kbn/licensing-types", + "@kbn/core-elasticsearch-server-utils", "@kbn/es-types", "@kbn/lazy-object" ], diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/retry_service/cases_analytics_retry_service.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/retry_service/cases_analytics_retry_service.ts index 450647dc29b96..31cae5c4974f6 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/retry_service/cases_analytics_retry_service.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/retry_service/cases_analytics_retry_service.ts @@ -7,7 +7,7 @@ import type { Logger } from '@kbn/core/server'; import type { errors as EsErrors } from '@elastic/elasticsearch'; -import { isRetryableEsClientError } from '../utils'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; import type { BackoffFactory } from '../../common/retry_service/types'; import { RetryService } from '../../common/retry_service'; diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/backfill_task_runner.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/backfill_task_runner.ts index f0b4c44c64df3..0c7d7dbb405c3 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/backfill_task_runner.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/backfill_task_runner.ts @@ -19,8 +19,8 @@ import type { IndicesGetMappingResponse, QueryDslQueryContainer, } from '@elastic/elasticsearch/lib/api/types'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; import type { ConfigType } from '../../../config'; -import { isRetryableEsClientError } from '../../utils'; interface BackfillTaskRunnerFactoryConstructorParams { taskInstance: ConcreteTaskInstance; diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/synchronization_sub_task.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/synchronization_sub_task.ts index 1f476d7ebccc4..8dcb49079af33 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/synchronization_sub_task.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/synchronization_sub_task.ts @@ -17,8 +17,8 @@ import type { IndicesGetMappingResponse, QueryDslQueryContainer, } from '@elastic/elasticsearch/lib/api/types'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; import type { Owner } from '../../../../common/constants/types'; -import { isRetryableEsClientError } from '../../utils'; import { type CAISyncType, SYNCHRONIZATION_QUERIES_DICTIONARY } from '../../constants'; const LOOKBACK_WINDOW = 5 * 60 * 1000; diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.test.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.test.ts deleted file mode 100644 index 40d82c070f9a8..0000000000000 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { errors as esErrors } from '@elastic/elasticsearch'; -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import { isRetryableEsClientError } from './utils'; - -describe('isRetryableEsClientError', () => { - describe('returns `false` for', () => { - test('non-retryable response errors', async () => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - body: { error: { type: 'cluster_block_exception' } }, - statusCode: 400, - }) - ); - - expect(isRetryableEsClientError(error)).toEqual(false); - }); - }); - - describe('returns `true` for', () => { - it('NoLivingConnectionsError', () => { - const error = new esErrors.NoLivingConnectionsError( - 'reason', - elasticsearchClientMock.createApiResponse() - ); - - expect(isRetryableEsClientError(error)).toEqual(true); - }); - - it('ConnectionError', () => { - const error = new esErrors.ConnectionError( - 'reason', - elasticsearchClientMock.createApiResponse() - ); - expect(isRetryableEsClientError(error)).toEqual(true); - }); - - it('TimeoutError', () => { - const error = new esErrors.TimeoutError( - 'reason', - elasticsearchClientMock.createApiResponse() - ); - expect(isRetryableEsClientError(error)).toEqual(true); - }); - - it('ResponseError of type snapshot_in_progress_exception', () => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - body: { error: { type: 'snapshot_in_progress_exception' } }, - }) - ); - expect(isRetryableEsClientError(error)).toEqual(true); - }); - - it.each([503, 504, 401, 403, 408, 410, 429])( - 'ResponseError with %p status code', - (statusCode) => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode, - body: { error: { type: 'reason' } }, - }) - ); - - expect(isRetryableEsClientError(error)).toEqual(true); - } - ); - }); -}); diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.ts index 00b13e8c6988f..236e6aab72ffa 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/utils.ts @@ -5,42 +5,9 @@ * 2.0. */ -import { errors as EsErrors } from '@elastic/elasticsearch'; import type { SavedObjectsClientContract } from '@kbn/core/server'; import { CASE_SAVED_OBJECT } from '../../common/constants'; -const retryResponseStatuses = [ - 401, // AuthorizationException - 403, // AuthenticationException - 408, // RequestTimeout - 410, // Gone - 429, // TooManyRequests -> ES circuit breaker - 503, // ServiceUnavailable - 504, // GatewayTimeout -]; - -/** - * Returns true if the given elasticsearch error should be retried - * by retry-based resiliency systems such as the SO migration, false otherwise. - */ -export const isRetryableEsClientError = (e: EsErrors.ElasticsearchClientError): boolean => { - if ( - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && - ((e?.statusCode && retryResponseStatuses.includes(e?.statusCode)) || - // ES returns a 400 Bad Request when trying to close or delete an - // index while snapshots are in progress. This should have been a 503 - // so once https://github.com/elastic/elasticsearch/issues/65883 is - // fixed we can remove this. - e?.body?.error?.type === 'snapshot_in_progress_exception')) - ) { - return true; - } - return false; -}; - const MAX_BUCKETS_LIMIT = 65535; export async function getAllSpacesWithCases(savedObjectsClient: SavedObjectsClientContract) { // This is one way to get all spaces that we want for case analytics purposes. diff --git a/x-pack/platform/plugins/shared/cases/tsconfig.json b/x-pack/platform/plugins/shared/cases/tsconfig.json index 4d45bfaa9fbe2..3177466a895d3 100644 --- a/x-pack/platform/plugins/shared/cases/tsconfig.json +++ b/x-pack/platform/plugins/shared/cases/tsconfig.json @@ -87,16 +87,16 @@ "@kbn/monaco", "@kbn/code-editor", "@kbn/logging", - "@kbn/core-elasticsearch-client-server-mocks", "@kbn/core-test-helpers-model-versions", "@kbn/alerting-types", "@kbn/elastic-assistant-common", "@kbn/test", "@kbn/dev-utils", "@kbn/tooling-log", + "@kbn/core-elasticsearch-server-utils", + "@kbn/core-notifications-browser-mocks", "@kbn/licensing-types", "@kbn/lazy-object", - "@kbn/core-notifications-browser-mocks", ], "exclude": [ "target/**/*" diff --git a/x-pack/platform/plugins/shared/entity_manager/server/lib/entities/helpers/retry.ts b/x-pack/platform/plugins/shared/entity_manager/server/lib/entities/helpers/retry.ts index c6ce2d9924fa3..84654f3378201 100644 --- a/x-pack/platform/plugins/shared/entity_manager/server/lib/entities/helpers/retry.ts +++ b/x-pack/platform/plugins/shared/entity_manager/server/lib/entities/helpers/retry.ts @@ -6,24 +6,12 @@ */ import { setTimeout } from 'timers/promises'; -import { errors as EsErrors } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/logging'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; import { EntitySecurityException } from '../errors/entity_security_exception'; const MAX_ATTEMPTS = 5; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: any) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - /** * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. * Should only be used to wrap operations that are idempotent and can be safely executed more than once. @@ -35,7 +23,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... diff --git a/x-pack/platform/plugins/shared/entity_manager/tsconfig.json b/x-pack/platform/plugins/shared/entity_manager/tsconfig.json index 9a7d532820fa9..a040fdb567498 100644 --- a/x-pack/platform/plugins/shared/entity_manager/tsconfig.json +++ b/x-pack/platform/plugins/shared/entity_manager/tsconfig.json @@ -35,5 +35,6 @@ "@kbn/core-saved-objects-server", "@kbn/features-plugin", "@kbn/zod-helpers", + "@kbn/core-elasticsearch-server-utils", ] } diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/helpers/retry.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/helpers/retry.ts index 8d57191476177..cfb47b82f16ca 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/helpers/retry.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/helpers/retry.ts @@ -6,23 +6,11 @@ */ import { setTimeout } from 'timers/promises'; -import { errors as EsErrors } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/logging'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 5; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: any) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - /** * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. * Should only be used to wrap operations that are idempotent and can be safely executed more than once. @@ -34,7 +22,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... diff --git a/x-pack/platform/plugins/shared/streams/tsconfig.json b/x-pack/platform/plugins/shared/streams/tsconfig.json index 6c9d1278efde3..f38220d94f37a 100644 --- a/x-pack/platform/plugins/shared/streams/tsconfig.json +++ b/x-pack/platform/plugins/shared/streams/tsconfig.json @@ -59,6 +59,7 @@ "@kbn/grok-heuristics", "@kbn/streamlang", "@kbn/licensing-types", + "@kbn/core-elasticsearch-server-utils", "@kbn/safer-lodash-set", "@kbn/sse-utils", "@kbn/global-search-plugin", diff --git a/x-pack/solutions/observability/plugins/observability/server/utils/retry.ts b/x-pack/solutions/observability/plugins/observability/server/utils/retry.ts index 421289d1c0479..8cf8662e5c89a 100644 --- a/x-pack/solutions/observability/plugins/observability/server/utils/retry.ts +++ b/x-pack/solutions/observability/plugins/observability/server/utils/retry.ts @@ -6,23 +6,11 @@ */ import { setTimeout } from 'timers/promises'; -import { errors as EsErrors } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/logging'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 5; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: any) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - /** * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. * Should only be used to wrap operations that are idempotent and can be safely executed more than once. @@ -34,7 +22,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... diff --git a/x-pack/solutions/observability/plugins/observability/tsconfig.json b/x-pack/solutions/observability/plugins/observability/tsconfig.json index 03be75809e969..26d860df5df20 100644 --- a/x-pack/solutions/observability/plugins/observability/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability/tsconfig.json @@ -132,6 +132,7 @@ "@kbn/deeplinks-management", "@kbn/content-management-content-editor", "@kbn/saved-objects-tagging-plugin", + "@kbn/core-elasticsearch-server-utils", "@kbn/core-http-browser-mocks", "@kbn/deeplinks-management", "@kbn/observability-nav-icons" diff --git a/x-pack/solutions/observability/plugins/slo/server/utils/retry.ts b/x-pack/solutions/observability/plugins/slo/server/utils/retry.ts index 421289d1c0479..8cf8662e5c89a 100644 --- a/x-pack/solutions/observability/plugins/slo/server/utils/retry.ts +++ b/x-pack/solutions/observability/plugins/slo/server/utils/retry.ts @@ -6,23 +6,11 @@ */ import { setTimeout } from 'timers/promises'; -import { errors as EsErrors } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/logging'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 5; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: any) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - /** * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. * Should only be used to wrap operations that are idempotent and can be safely executed more than once. @@ -34,7 +22,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... diff --git a/x-pack/solutions/observability/plugins/slo/tsconfig.json b/x-pack/solutions/observability/plugins/slo/tsconfig.json index 26dee16dbe1a9..6d0322dd4bc3a 100644 --- a/x-pack/solutions/observability/plugins/slo/tsconfig.json +++ b/x-pack/solutions/observability/plugins/slo/tsconfig.json @@ -109,6 +109,7 @@ "@kbn/lock-manager", "@kbn/object-utils", "@kbn/licensing-types", + "@kbn/core-elasticsearch-server-utils", "@kbn/deeplinks-analytics", "@kbn/content-management-plugin", "@kbn/dashboards-selector", diff --git a/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.test.ts b/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.test.ts index 9120bdab51866..52db1086b8a50 100644 --- a/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.test.ts +++ b/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.test.ts @@ -52,7 +52,7 @@ describe('retryTransientEsErrors', () => { }); it('should throw non-transient errors', async () => { - const error = new EsErrors.ResponseError({ statusCode: 429 } as DiagnosticResult); + const error = new EsErrors.ResponseError({ statusCode: 403 } as DiagnosticResult); const mockFn = jest.fn(); mockFn.mockRejectedValueOnce(error); diff --git a/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.ts b/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.ts index 6dd568de3b98c..ea2a1aa9dd618 100644 --- a/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.ts +++ b/x-pack/solutions/security/packages/index-adapter/src/retry_transient_es_errors.ts @@ -6,24 +6,10 @@ */ import type { Logger } from '@kbn/core/server'; -import { errors as EsErrors } from '@elastic/elasticsearch'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 3; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: Error) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && - e?.statusCode && - retryResponseStatuses.includes(e.statusCode)); - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const retryTransientEsErrors = async ( @@ -33,7 +19,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... diff --git a/x-pack/solutions/security/packages/index-adapter/tsconfig.json b/x-pack/solutions/security/packages/index-adapter/tsconfig.json index fb98020940ad0..481a518c5ce89 100644 --- a/x-pack/solutions/security/packages/index-adapter/tsconfig.json +++ b/x-pack/solutions/security/packages/index-adapter/tsconfig.json @@ -13,6 +13,7 @@ "@kbn/std", "@kbn/safer-lodash-set", "@kbn/logging-mocks", + "@kbn/core-elasticsearch-server-utils", ], "exclude": [ "target/**/*" diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/retry_transient_es_errors.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/retry_transient_es_errors.ts index 7a3839ad3c5bc..adfd3eab0bbfb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/retry_transient_es_errors.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/retry_transient_es_errors.ts @@ -6,23 +6,10 @@ */ import type { Logger } from '@kbn/core/server'; -import { errors as EsErrors } from '@elastic/elasticsearch'; +import { isRetryableEsClientError } from '@kbn/core-elasticsearch-server-utils'; const MAX_ATTEMPTS = 3; -const retryResponseStatuses = [ - 503, // ServiceUnavailable - 408, // RequestTimeout - 410, // Gone -]; - -const isRetryableError = (e: Error) => - e instanceof EsErrors.NoLivingConnectionsError || - e instanceof EsErrors.ConnectionError || - e instanceof EsErrors.TimeoutError || - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const retryTransientEsErrors = async ( @@ -38,7 +25,7 @@ export const retryTransientEsErrors = async ( try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableEsClientError(e)) { const retryCount = attempt + 1; const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... diff --git a/x-pack/solutions/security/plugins/security_solution/tsconfig.json b/x-pack/solutions/security/plugins/security_solution/tsconfig.json index f09f2a49f9b74..ba21a65dc89e5 100644 --- a/x-pack/solutions/security/plugins/security_solution/tsconfig.json +++ b/x-pack/solutions/security/plugins/security_solution/tsconfig.json @@ -259,6 +259,7 @@ "@kbn/core-lifecycle-server-mocks", "@kbn/licensing-types", "@kbn/core-metrics-server", + "@kbn/core-elasticsearch-server-utils", "@kbn/rrule", "@kbn/onechat-genai-utils", "@kbn/onechat-server", diff --git a/yarn.lock b/yarn.lock index b3a774355bb58..7164edcd91461 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5021,6 +5021,10 @@ version "0.0.0" uid "" +"@kbn/core-elasticsearch-server-utils@link:src/core/packages/elasticsearch/server-utils": + version "0.0.0" + uid "" + "@kbn/core-elasticsearch-server@link:src/core/packages/elasticsearch/server": version "0.0.0" uid ""