From 30eb4828c99c588c05403b6e0c157db217a836a8 Mon Sep 17 00:00:00 2001 From: Sonia Sanz Vivas Date: Tue, 14 Apr 2026 16:24:36 +0200 Subject: [PATCH 1/2] Transfer retention tab to Kibana Management --- .github/CODEOWNERS | 11 +- .../plugins/shared/streams/common/index.ts | 1 + .../lib/streams/failure_store/index.ts} | 2 +- .../streams/failure_store/route_helpers.ts | 153 ++++++++++++++++++ .../streams/server/lib/streams/stream_crud.ts | 145 ----------------- .../internal/streams/failure_store/route.ts | 2 +- .../helpers/failure_store_index_name.ts | 2 +- 7 files changed, 166 insertions(+), 150 deletions(-) rename x-pack/platform/plugins/shared/{streams_app/public/util/constants.ts => streams/server/lib/streams/failure_store/index.ts} (71%) create mode 100644 x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/route_helpers.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 193af296d4256..e39e18b4c2acb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1729,8 +1729,15 @@ x-pack/solutions/observability/plugins/observability/server/lib/esql_extensions ## Streams parts owned by Logs UX /x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management @elastic/obs-onboarding-team -# Streams parts owned by Kibana Management -x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle/downsampling @elastic/kibana-management +# Streams - Data retention overrides (owned by Kibana Management) +x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/routes/internal/streams/failure_store @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/routes/internal/streams/lifecycle @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/lib/streams/lifecycle/ilm_policies.ts @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/lib/streams/lifecycle/ilm_policies.test.ts @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/lib/streams/lifecycle/ilm_phases.ts @elastic/kibana-management +x-pack/platform/plugins/shared/streams/server/lib/streams/lifecycle/ilm_phases.test.ts @elastic/kibana-management # Streams — sig events overrides (obs-sig-events-team owns these subtrees) x-pack/platform/plugins/shared/streams_app/public/components/sig_events @elastic/obs-sig-events-team diff --git a/x-pack/platform/plugins/shared/streams/common/index.ts b/x-pack/platform/plugins/shared/streams/common/index.ts index 8a828e55b2cd3..10086ae937615 100644 --- a/x-pack/platform/plugins/shared/streams/common/index.ts +++ b/x-pack/platform/plugins/shared/streams/common/index.ts @@ -21,6 +21,7 @@ export { ATTACHMENT_SUGGESTIONS_LIMIT, DEFAULT_EXTRACTION_INTERVAL_HOURS, MIN_EXTRACTION_INTERVAL_HOURS, + FAILURE_STORE_SELECTOR, } from './constants'; export type { StreamDocsStat } from './doc_counts'; diff --git a/x-pack/platform/plugins/shared/streams_app/public/util/constants.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/index.ts similarity index 71% rename from x-pack/platform/plugins/shared/streams_app/public/util/constants.ts rename to x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/index.ts index f3bcf1881a22a..27694d98e5647 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/util/constants.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const FAILURE_STORE_SELECTOR = '::failures'; +export { getClusterDefaultFailureStoreRetentionValue, getFailureStoreStats } from './route_helpers'; diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/route_helpers.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/route_helpers.ts new file mode 100644 index 0000000000000..7d3f85d3bd3e3 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/failure_store/route_helpers.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DocStats, EpochTime, UnitMillis } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { FailureStoreStatsResponse } from '@kbn/streams-schema/src/models/ingest/failure_store'; + +import { FAILURE_STORE_SELECTOR } from '../../../../common/constants'; +import { parseError } from '../errors/parse_error'; + +export async function getClusterDefaultFailureStoreRetentionValue({ + esClient, + isServerless, +}: { + esClient: ElasticsearchClient; + isServerless: boolean; +}): Promise { + let defaultRetention: string | undefined; + try { + if (!isServerless) { + const { persistent, defaults } = await esClient.cluster.getSettings({ + include_defaults: true, + }); + const persistentDSRetention = + persistent?.data_streams?.lifecycle?.retention?.failures_default; + const defaultsDSRetention = defaults?.data_streams?.lifecycle?.retention?.failures_default; + defaultRetention = persistentDSRetention ?? defaultsDSRetention; + } + } catch (e) { + const { statusCode } = parseError(e); + if (statusCode === 403) { + // if user doesn't have permissions to read cluster settings, we just return undefined + } else { + throw e; + } + } + return defaultRetention; +} + +export async function getFailureStoreStats({ + name, + esClient, + esClientAsSecondaryAuthUser, + isServerless, +}: { + name: string; + esClient: ElasticsearchClient; + esClientAsSecondaryAuthUser?: ElasticsearchClient; + isServerless: boolean; +}): Promise { + const failureStoreDocs = + isServerless && esClientAsSecondaryAuthUser + ? await getFailureStoreMeteringSize({ name, esClientAsSecondaryAuthUser }) + : await getFailureStoreSize({ name, esClient }); + const creationDate = await getFailureStoreCreationDate({ name, esClient }); + + return { + size: failureStoreDocs?.total_size_in_bytes, + count: failureStoreDocs?.count, + creationDate, + }; +} + +async function getFailureStoreSize({ + name, + esClient, +}: { + name: string; + esClient: ElasticsearchClient; +}): Promise { + try { + const response = await esClient.indices.stats({ + index: `${name}${FAILURE_STORE_SELECTOR}`, + metric: ['docs'], + forbid_closed_indices: false, + }); + const docsStats = response?._all?.total?.docs; + return { + count: docsStats?.count || 0, + total_size_in_bytes: docsStats?.total_size_in_bytes || 0, + }; + } catch (e) { + const { statusCode } = parseError(e); + if (statusCode === 404) { + return undefined; + } else { + throw e; + } + } +} + +async function getFailureStoreMeteringSize({ + name, + esClientAsSecondaryAuthUser, +}: { + name: string; + esClientAsSecondaryAuthUser: ElasticsearchClient; +}): Promise { + try { + const response = await esClientAsSecondaryAuthUser.transport.request<{ + _total: { num_docs: number; size_in_bytes: number }; + }>({ + method: 'GET', + path: `/_metering/stats/${name}${FAILURE_STORE_SELECTOR}`, + }); + + return { + count: response._total?.num_docs || 0, + total_size_in_bytes: response._total?.size_in_bytes || 0, + }; + } catch (e) { + const { statusCode } = parseError(e); + if (statusCode === 404) { + return undefined; + } else { + throw e; + } + } +} + +async function getFailureStoreCreationDate({ + name, + esClient, +}: { + name: string; + esClient: ElasticsearchClient; +}): Promise { + let age: number | undefined; + try { + const response = await esClient.indices.explainDataLifecycle({ + index: `${name}${FAILURE_STORE_SELECTOR}`, + }); + const indices = response.indices; + if (indices && typeof indices === 'object') { + const firstIndex = Object.values(indices)[0] as { + index_creation_date_millis?: EpochTime; + }; + age = firstIndex?.index_creation_date_millis; + } + return age || undefined; + } catch (e) { + const { statusCode } = parseError(e); + if (statusCode === 404) { + return undefined; + } else { + throw e; + } + } +} diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/stream_crud.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/stream_crud.ts index 40ad46554fc7c..e4979447736ba 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/stream_crud.ts @@ -7,18 +7,14 @@ import type { ClusterComponentTemplate, - DocStats, - EpochTime, IndicesDataStream, IndicesGetDataStreamSettingsDataStreamSettings, IndicesGetIndexTemplateIndexTemplateItem, IngestPipeline, - UnitMillis, } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { EffectiveFailureStore, - FailureStoreStatsResponse, DataStreamWithFailureStore, } from '@kbn/streams-schema/src/models/ingest/failure_store'; import type { @@ -27,7 +23,6 @@ import type { } from '@kbn/streams-schema'; import type { DownsampleStep } from '@kbn/streams-schema/src/models/ingest/lifecycle'; -import { FAILURE_STORE_SELECTOR } from '../../../common/constants'; import { DefinitionNotFoundError } from './errors/definition_not_found_error'; import { parseError } from './errors/parse_error'; @@ -313,35 +308,6 @@ export async function getDataStream({ return dataStream; } -export async function getClusterDefaultFailureStoreRetentionValue({ - esClient, - isServerless, -}: { - esClient: ElasticsearchClient; - isServerless: boolean; -}): Promise { - let defaultRetention: string | undefined; - try { - if (!isServerless) { - const { persistent, defaults } = await esClient.cluster.getSettings({ - include_defaults: true, - }); - const persistentDSRetention = - persistent?.data_streams?.lifecycle?.retention?.failures_default; - const defaultsDSRetention = defaults?.data_streams?.lifecycle?.retention?.failures_default; - defaultRetention = persistentDSRetention ?? defaultsDSRetention; - } - } catch (e) { - const { statusCode } = parseError(e); - if (statusCode === 403) { - // if user doesn't have permissions to read cluster settings, we just return undefined - } else { - throw e; - } - } - return defaultRetention; -} - export function getFailureStore({ dataStream, }: { @@ -377,114 +343,3 @@ export function getFailureStore({ return { disabled: {} }; } - -export async function getFailureStoreStats({ - name, - esClient, - esClientAsSecondaryAuthUser, - isServerless, -}: { - name: string; - esClient: ElasticsearchClient; - esClientAsSecondaryAuthUser?: ElasticsearchClient; - isServerless: boolean; -}): Promise { - const failureStoreDocs = - isServerless && esClientAsSecondaryAuthUser - ? await getFailureStoreMeteringSize({ name, esClientAsSecondaryAuthUser }) - : await getFailureStoreSize({ name, esClient }); - const creationDate = await getFailureStoreCreationDate({ name, esClient }); - - return { - size: failureStoreDocs?.total_size_in_bytes, - count: failureStoreDocs?.count, - creationDate, - }; -} - -export async function getFailureStoreSize({ - name, - esClient, -}: { - name: string; - esClient: ElasticsearchClient; -}): Promise { - try { - const response = await esClient.indices.stats({ - index: `${name}${FAILURE_STORE_SELECTOR}`, - metric: ['docs'], - forbid_closed_indices: false, - }); - const docsStats = response?._all?.total?.docs; - return { - count: docsStats?.count || 0, - total_size_in_bytes: docsStats?.total_size_in_bytes || 0, - }; - } catch (e) { - const { statusCode } = parseError(e); - if (statusCode === 404) { - return undefined; - } else { - throw e; - } - } -} - -export async function getFailureStoreMeteringSize({ - name, - esClientAsSecondaryAuthUser, -}: { - name: string; - esClientAsSecondaryAuthUser: ElasticsearchClient; -}): Promise { - try { - const response = await esClientAsSecondaryAuthUser.transport.request<{ - _total: { num_docs: number; size_in_bytes: number }; - }>({ - method: 'GET', - path: `/_metering/stats/${name}${FAILURE_STORE_SELECTOR}`, - }); - - return { - count: response._total?.num_docs || 0, - total_size_in_bytes: response._total?.size_in_bytes || 0, - }; - } catch (e) { - const { statusCode } = parseError(e); - if (statusCode === 404) { - return undefined; - } else { - throw e; - } - } -} - -export async function getFailureStoreCreationDate({ - name, - esClient, -}: { - name: string; - esClient: ElasticsearchClient; -}): Promise { - let age: number | undefined; - try { - const response = await esClient.indices.explainDataLifecycle({ - index: `${name}${FAILURE_STORE_SELECTOR}`, - }); - const indices = response.indices; - if (indices && typeof indices === 'object') { - const firstIndex = Object.values(indices)[0] as { - index_creation_date_millis?: EpochTime; - }; - age = firstIndex?.index_creation_date_millis; - } - return age || undefined; - } catch (e) { - const { statusCode } = parseError(e); - if (statusCode === 404) { - return undefined; - } else { - throw e; - } - } -} diff --git a/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/failure_store/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/failure_store/route.ts index 87e8619a47ebb..768b63ee0702d 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/failure_store/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/failure_store/route.ts @@ -9,7 +9,7 @@ import { z } from '@kbn/zod/v4'; import { getClusterDefaultFailureStoreRetentionValue, getFailureStoreStats, -} from '../../../../lib/streams/stream_crud'; +} from '../../../../lib/streams/failure_store'; import { STREAMS_API_PRIVILEGES } from '../../../../../common/constants'; import { createServerRoute } from '../../../create_server_route'; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle/helpers/failure_store_index_name.ts b/x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle/helpers/failure_store_index_name.ts index 7cd728d4bb4d6..1429e51b23112 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle/helpers/failure_store_index_name.ts +++ b/x-pack/platform/plugins/shared/streams_app/public/components/stream_management/data_management/stream_detail_lifecycle/helpers/failure_store_index_name.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { FAILURE_STORE_SELECTOR } from '../../../../../util/constants'; +import { FAILURE_STORE_SELECTOR } from '@kbn/streams-plugin/common'; export const getFailureStoreIndexName = (streamName: string) => { return streamName + FAILURE_STORE_SELECTOR; From 0bc25bc3a507556947eac8731268342bfd34ca43 Mon Sep 17 00:00:00 2001 From: Sonia Sanz Vivas Date: Mon, 20 Apr 2026 09:30:10 +0200 Subject: [PATCH 2/2] Update bundle limit --- .review/worktrees/283dd855 | 1 + packages/kbn-optimizer/limits.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 .review/worktrees/283dd855 diff --git a/.review/worktrees/283dd855 b/.review/worktrees/283dd855 new file mode 160000 index 0000000000000..283dd8559f9ce --- /dev/null +++ b/.review/worktrees/283dd855 @@ -0,0 +1 @@ +Subproject commit 283dd8559f9ce211190912ddace732919fefa02c diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index dde0248f251c2..04982c4ce0721 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -176,7 +176,7 @@ pageLoadAssetSize: spaces: 28871 stackAlerts: 31499 stackConnectors: 85421 - streams: 14000 + streams: 15434 streamsApp: 25375 synthetics: 31571 telemetry: 25755