From ba10f89f8f917719eb5038e5b63dd2070e57b3a8 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 16 Dec 2025 16:25:48 +0100 Subject: [PATCH] [Infra] Fix legacy Infra search client to respect the frozen tier setting (#246438) ## Summary Basically what this https://github.com/elastic/kibana/pull/245224 does, but focusing only on the Infra plugin changes `has_data` endpoint will run this when the setting is not set ```json { "index": "remote_cluster:metrics-*,remote_cluster:metricbeat-*", "allow_no_indices": true, "terminate_after": 1, "ignore_unavailable": true, "body": { "size": 0, "query": { "bool": {} } } } ``` and this when the setting is set ```json params { "index": "remote_cluster:metrics-*,remote_cluster:metricbeat-*", "allow_no_indices": true, "terminate_after": 1, "ignore_unavailable": true, "body": { "size": 0, "query": { "bool": { "filter": [ { "bool": { "must_not": [ { "terms": { "_tier": [ "data_frozen" ] } } ] } } ] } } } } ``` Other pages are affected by this change, but are not a source for the frozen tier being hit because they don't run unbounded queries - Infra Inventory UI ```json params { "allow_no_indices": true, "ignore_unavailable": true, "index": "remote_cluster:metrics-*,remote_cluster:metricbeat-*", "body": { "size": 0, "query": { "bool": { "must": [ { "bool": { "filter": [ { "bool": { "filter": [ { "bool": { "filter": [ { "term": { "data_stream.dataset": "hostmetricsreceiver.otel" } } ] } } ] } }, { "range": { "@timestamp": { "gte": 1765824423511, "lte": 1765824723511, "format": "epoch_millis" } } }, { "exists": { "field": "host.name" } } ] } } ], "filter": [ { "bool": { "must_not": [ { "terms": { "_tier": [ "data_frozen" ] } } ] } } ] } }, "aggs": { "groupings": { "composite": { "size": 2000, "sources": [ { "groupBy0": { "terms": { "field": "host.name" } } } ] }, "aggs": { "histogram": { "date_histogram": { "field": "@timestamp", "fixed_interval": "60s", "offset": "-57511ms", "extended_bounds": { "min": 1765824423511, "max": 1765824723511 } }, "aggregations": { "cpu_idle": { "terms": { "field": "state", "include": [ "idle", "wait" ] }, "aggs": { "avg": { "avg": { "field": "system.cpu.utilization" } } } }, "cpu_idle_total": { "sum_bucket": { "buckets_path": "cpu_idle.avg" } }, "cpuV2": { "bucket_script": { "buckets_path": { "cpuIdleTotal": "cpu_idle_total" }, "script": "1 - params.cpuIdleTotal", "gap_policy": "skip" } }, "__metadata__": { "top_metrics": { "size": 1, "metrics": [ { "field": "host.name" }, { "field": "host.ip" }, { "field": "host.os.name" }, { "field": "cloud.provider" } ], "sort": { "@timestamp": "desc" } } } } }, "metricsets": { "terms": { "field": "metricset.name" } } } } } } } ``` - Metrics Explorer ```json { "allow_no_indices": true, "ignore_unavailable": true, "index": "remote_cluster:metrics-*,remote_cluster:metricbeat-*", "body": { "size": 0, "query": { "bool": { "must": [ { "bool": { "filter": [ null, { "range": { "@timestamp": { "gte": 1765821298325, "lte": 1765824898325, "format": "epoch_millis" } } } ] } } ], "filter": [ { "bool": { "must_not": [ { "terms": { "_tier": [ "data_frozen" ] } } ] } } ] } }, "aggs": { "histogram": { "date_histogram": { "field": "@timestamp", "fixed_interval": "30s", "offset": "0s", "extended_bounds": { "min": 1765821298325, "max": 1765824898325 } }, "aggregations": { "metric_0": { "avg": { "field": "system.cpu.total.norm.pct" } }, "metric_1": { "avg": { "field": "kubernetes.pod.cpu.usage.node.pct" } }, "metric_2": { "avg": { "field": "docker.cpu.total.pct" } } } }, "metricsets": { "terms": { "field": "metricset.name" } } } } } ``` --------- Co-authored-by: Nathan L Smith Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Caue Marcondes Co-authored-by: Elastic Machine Co-authored-by: Coen Warmer (cherry picked from commit aedfb94b2118c1e925e5678c585ab2093b04ed12) # Conflicts: # x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts # x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts # x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts # x-pack/solutions/observability/plugins/metrics_data_access/moon.yml # x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json --- .../services/log_views/exclude_tiers_query.ts | 26 ++++++++++++ .../services/log_views/log_views_client.ts | 20 +++++++-- .../public/services/log_views/types.ts | 16 +++++--- .../public/utils/logs_overview_fetchers.ts | 41 +++++++++++++------ .../infra/server/lib/create_search_client.ts | 34 +++++++++++++-- .../server/routes/metrics_sources/index.ts | 2 +- .../infra/server/routes/overview/index.ts | 2 +- .../infra/server/routes/snapshot/index.ts | 2 +- .../server/lib/create_search_client.ts | 32 +++++++++++++-- .../server/routes/metrics_explorer/index.ts | 2 +- .../plugins/metrics_data_access/tsconfig.json | 3 +- 11 files changed, 145 insertions(+), 35 deletions(-) create mode 100644 x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts new file mode 100644 index 0000000000000..16bb9e24f505a --- /dev/null +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +export function excludeTiersQuery( + excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'> +): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + terms: { + _tier: excludedDataTiers, + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts index 7fc34a1035114..04128c25535e5 100644 --- a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { HttpStart } from '@kbn/core/public'; +import type { HttpStart, IUiSettingsClient } from '@kbn/core/public'; import type { ISearchGeneric } from '@kbn/search-types'; import { DataViewsContract } from '@kbn/data-views-plugin/public'; import { lastValueFrom } from 'rxjs'; @@ -28,7 +28,8 @@ import { isNoSuchRemoteClusterError, } from '../../../common/log_views'; import { decodeOrThrow } from '../../../common/runtime_types'; -import { ILogViewsClient } from './types'; +import type { ILogViewsClient } from './types'; +import { excludeTiersQuery } from './exclude_tiers_query'; export class LogViewsClient implements ILogViewsClient { constructor( @@ -69,8 +70,16 @@ export class LogViewsClient implements ILogViewsClient { return resolvedLogView; } - public async getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise { - return await lastValueFrom( + public async getResolvedLogViewStatus( + resolvedLogView: ResolvedLogView, + uiSettings?: IUiSettingsClient + ): Promise { + const excludedDataTiers = uiSettings?.get('observability:searchExcludedDataTiers') ?? []; + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + const indexStatus = await lastValueFrom( this.search({ params: { ignore_unavailable: true, @@ -79,6 +88,7 @@ export class LogViewsClient implements ILogViewsClient { size: 0, terminate_after: 1, track_total_hits: 1, + query: excludedQuery ? { bool: { filter: excludedQuery } } : undefined, }, }) ).then( @@ -119,6 +129,8 @@ export class LogViewsClient implements ILogViewsClient { ); } ); + + return indexStatus; } public async putLogView( diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts index a6d516f00669d..d28adc6e02b2d 100644 --- a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { HttpStart } from '@kbn/core/public'; -import { ISearchStart } from '@kbn/data-plugin/public'; -import { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; -import { +import type { HttpStart } from '@kbn/core/public'; +import type { ISearchStart } from '@kbn/data-plugin/public'; +import type { DataViewsContract } from '@kbn/data-views-plugin/public'; +import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import type { LogView, LogViewAttributes, LogViewReference, @@ -35,7 +36,10 @@ export interface LogViewsServiceStartDeps { export interface ILogViewsClient { getLogView(logViewReference: LogViewReference): Promise; - getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise; + getResolvedLogViewStatus( + resolvedLogView: ResolvedLogView, + uiSettings?: IUiSettingsClient + ): Promise; getResolvedLogView(logViewReference: LogViewReference): Promise; putLogView( logViewReference: LogViewReference, diff --git a/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts index e8afe5769ca5d..14e3fe3c2fc7d 100644 --- a/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -12,6 +12,8 @@ import type { LogsFetchDataResponse, } from '@kbn/observability-plugin/public'; import { DEFAULT_LOG_VIEW, getLogsLocatorsFromUrlService } from '@kbn/logs-shared-plugin/common'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; import { TIMESTAMP_FIELD } from '../../common/constants'; import type { InfraClientStartDeps, InfraClientStartServicesAccessor } from '../types'; @@ -37,10 +39,11 @@ type StatsAndSeries = Pick; export function getLogsHasDataFetcher(getStartServices: InfraClientStartServicesAccessor) { return async () => { - const [, { logsShared }] = await getStartServices(); + const [coreStart, { logsShared }] = await getStartServices(); const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW); const logViewStatus = await logsShared.logViews.client.getResolvedLogViewStatus( - resolvedLogView + resolvedLogView, + coreStart.uiSettings ); const hasData = logViewStatus.index === 'available'; @@ -57,7 +60,7 @@ export function getLogsOverviewDataFetcher( getStartServices: InfraClientStartServicesAccessor ): FetchData { return async (params) => { - const [, { data, logsShared, share }] = await getStartServices(); + const [coreStart, { data, logsShared, share }] = await getStartServices(); const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW); const { stats, series } = await fetchLogsOverview( @@ -65,7 +68,8 @@ export function getLogsOverviewDataFetcher( index: resolvedLogView.indices, }, params, - data + data, + coreStart.uiSettings ); const { logsLocator } = getLogsLocatorsFromUrlService(share.url); const timeSpanInMinutes = (params.absoluteTime.end - params.absoluteTime.start) / (1000 * 60); @@ -89,8 +93,12 @@ export function getLogsOverviewDataFetcher( async function fetchLogsOverview( logParams: LogParams, params: FetchDataParams, - dataPlugin: InfraClientStartDeps['data'] + dataPlugin: InfraClientStartDeps['data'], + uiSettings: IUiSettingsClient ): Promise { + const excludedDataTiers = + uiSettings?.get('observability:searchExcludedDataTiers') ?? []; + return new Promise((resolve, reject) => { let esResponse: estypes.SearchResponse | undefined; @@ -100,7 +108,7 @@ async function fetchLogsOverview( index: logParams.index, body: { size: 0, - query: buildLogOverviewQuery(logParams, params), + query: buildLogOverviewQuery(params, excludedDataTiers), aggs: buildLogOverviewAggregations(logParams, params), }, }, @@ -119,14 +127,21 @@ async function fetchLogsOverview( }); } -function buildLogOverviewQuery(logParams: LogParams, params: FetchDataParams) { +function buildLogOverviewQuery(params: FetchDataParams, excludedDataTiers: DataTier[]) { return { - range: { - [TIMESTAMP_FIELD]: { - gt: new Date(params.absoluteTime.start).toISOString(), - lte: new Date(params.absoluteTime.end).toISOString(), - format: 'strict_date_optional_time', - }, + bool: { + must: [ + { + range: { + [TIMESTAMP_FIELD]: { + gt: new Date(params.absoluteTime.start).toISOString(), + lte: new Date(params.absoluteTime.end).toISOString(), + format: 'strict_date_optional_time', + }, + }, + }, + ], + must_not: excludedDataTiers.length ? [{ terms: { _tier: excludedDataTiers } }] : undefined, }, }; } diff --git a/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts b/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts index e2e5cd93a6154..3344865503714 100644 --- a/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts +++ b/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts @@ -6,6 +6,9 @@ */ import type { KibanaRequest } from '@kbn/core/server'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { InfraPluginRequestHandlerContext } from '../types'; import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter'; @@ -16,7 +19,32 @@ export const createSearchClient = framework: KibanaFramework, request?: KibanaRequest ) => - ( + async ( opts: CallWithRequestParams - ): Promise> => - framework.callWithRequest(requestContext, 'search', opts, request); + ): Promise> => { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + return framework.callWithRequest( + requestContext, + 'search', + { + ...opts, + body: { + ...(opts.body ?? {}), + query: { + bool: { + ...(opts.body?.query ? { must: [opts.body?.query] } : {}), + filter: excludedQuery, + }, + }, + }, + }, + request + ); + }; diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts index 4290db429aead..7fda86a9baa6e 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts @@ -186,7 +186,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => async (requestContext, request, response) => { const { sourceId } = request.params; - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const soClient = (await requestContext.core).savedObjects.client; const source = await libs.sources.getSourceConfiguration(soClient, sourceId); diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts index 0e33c8733b59c..c5b25a24601a9 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts @@ -23,7 +23,7 @@ export const initOverviewRoute = (libs: InfraBackendLibs) => { }, async (requestContext, request, response) => { const options = request.body; - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const soClient = (await requestContext.core).savedObjects.client; const source = await libs.sources.getSourceConfiguration(soClient, options.sourceId); diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts index a6f7f998572b4..980f591825d48 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts @@ -57,7 +57,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => { ); UsageCollector.countNode(snapshotRequest.nodeType); - const client = createSearchClient(requestContext, framework, request); + const client = await createSearchClient(requestContext, framework, request); const snapshotResponse = await getNodes( client, diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts b/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts index 2fa4f887e9983..3a2e466a29463 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts +++ b/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts @@ -6,12 +6,36 @@ */ import type { RequestHandlerContext } from '@kbn/core/server'; -import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter'; +import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; export const createSearchClient = (requestContext: RequestHandlerContext, framework: KibanaFramework) => - ( + async ( opts: CallWithRequestParams - ): Promise> => - framework.callWithRequest(requestContext, 'search', opts); + ): Promise> => { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get( + 'observability:searchExcludedDataTiers' + ); + + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + return framework.callWithRequest(requestContext, 'search', { + ...opts, + body: { + ...(opts.body ?? {}), + query: { + bool: { + ...(opts.body?.query ? { must: [opts.body?.query] } : {}), + filter: excludedQuery, + }, + }, + }, + }); + }; diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts b/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts index 44392684d6566..1d2de6b65667b 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts +++ b/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts @@ -41,7 +41,7 @@ export const initMetricExplorerRoute = (framework: KibanaFramework) => { ); } - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const interval = await findIntervalForMetrics(client, options); const optionsWithInterval = options.forceInterval diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json b/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json index 2d995041b8661..0fe8d1e15a373 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json +++ b/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json @@ -35,6 +35,7 @@ "@kbn/lens-embeddable-utils", "@kbn/react-kibana-context-render", "@kbn/react-kibana-context-theme", - "@kbn/router-utils" + "@kbn/router-utils", + "@kbn/observability-utils-common" ] }