diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts new file mode 100644 index 0000000000000..16bb9e24f505a --- /dev/null +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +export function excludeTiersQuery( + excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'> +): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + terms: { + _tier: excludedDataTiers, + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts index 7fc34a1035114..04128c25535e5 100644 --- a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { HttpStart } from '@kbn/core/public'; +import type { HttpStart, IUiSettingsClient } from '@kbn/core/public'; import type { ISearchGeneric } from '@kbn/search-types'; import { DataViewsContract } from '@kbn/data-views-plugin/public'; import { lastValueFrom } from 'rxjs'; @@ -28,7 +28,8 @@ import { isNoSuchRemoteClusterError, } from '../../../common/log_views'; import { decodeOrThrow } from '../../../common/runtime_types'; -import { ILogViewsClient } from './types'; +import type { ILogViewsClient } from './types'; +import { excludeTiersQuery } from './exclude_tiers_query'; export class LogViewsClient implements ILogViewsClient { constructor( @@ -69,8 +70,16 @@ export class LogViewsClient implements ILogViewsClient { return resolvedLogView; } - public async getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise { - return await lastValueFrom( + public async getResolvedLogViewStatus( + resolvedLogView: ResolvedLogView, + uiSettings?: IUiSettingsClient + ): Promise { + const excludedDataTiers = uiSettings?.get('observability:searchExcludedDataTiers') ?? []; + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + const indexStatus = await lastValueFrom( this.search({ params: { ignore_unavailable: true, @@ -79,6 +88,7 @@ export class LogViewsClient implements ILogViewsClient { size: 0, terminate_after: 1, track_total_hits: 1, + query: excludedQuery ? { bool: { filter: excludedQuery } } : undefined, }, }) ).then( @@ -119,6 +129,8 @@ export class LogViewsClient implements ILogViewsClient { ); } ); + + return indexStatus; } public async putLogView( diff --git a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts index a6d516f00669d..d28adc6e02b2d 100644 --- a/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts +++ b/x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { HttpStart } from '@kbn/core/public'; -import { ISearchStart } from '@kbn/data-plugin/public'; -import { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; -import { +import type { HttpStart } from '@kbn/core/public'; +import type { ISearchStart } from '@kbn/data-plugin/public'; +import type { DataViewsContract } from '@kbn/data-views-plugin/public'; +import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import type { LogView, LogViewAttributes, LogViewReference, @@ -35,7 +36,10 @@ export interface LogViewsServiceStartDeps { export interface ILogViewsClient { getLogView(logViewReference: LogViewReference): Promise; - getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise; + getResolvedLogViewStatus( + resolvedLogView: ResolvedLogView, + uiSettings?: IUiSettingsClient + ): Promise; getResolvedLogView(logViewReference: LogViewReference): Promise; putLogView( logViewReference: LogViewReference, diff --git a/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts index e8afe5769ca5d..14e3fe3c2fc7d 100644 --- a/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -12,6 +12,8 @@ import type { LogsFetchDataResponse, } from '@kbn/observability-plugin/public'; import { DEFAULT_LOG_VIEW, getLogsLocatorsFromUrlService } from '@kbn/logs-shared-plugin/common'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; import { TIMESTAMP_FIELD } from '../../common/constants'; import type { InfraClientStartDeps, InfraClientStartServicesAccessor } from '../types'; @@ -37,10 +39,11 @@ type StatsAndSeries = Pick; export function getLogsHasDataFetcher(getStartServices: InfraClientStartServicesAccessor) { return async () => { - const [, { logsShared }] = await getStartServices(); + const [coreStart, { logsShared }] = await getStartServices(); const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW); const logViewStatus = await logsShared.logViews.client.getResolvedLogViewStatus( - resolvedLogView + resolvedLogView, + coreStart.uiSettings ); const hasData = logViewStatus.index === 'available'; @@ -57,7 +60,7 @@ export function getLogsOverviewDataFetcher( getStartServices: InfraClientStartServicesAccessor ): FetchData { return async (params) => { - const [, { data, logsShared, share }] = await getStartServices(); + const [coreStart, { data, logsShared, share }] = await getStartServices(); const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW); const { stats, series } = await fetchLogsOverview( @@ -65,7 +68,8 @@ export function getLogsOverviewDataFetcher( index: resolvedLogView.indices, }, params, - data + data, + coreStart.uiSettings ); const { logsLocator } = getLogsLocatorsFromUrlService(share.url); const timeSpanInMinutes = (params.absoluteTime.end - params.absoluteTime.start) / (1000 * 60); @@ -89,8 +93,12 @@ export function getLogsOverviewDataFetcher( async function fetchLogsOverview( logParams: LogParams, params: FetchDataParams, - dataPlugin: InfraClientStartDeps['data'] + dataPlugin: InfraClientStartDeps['data'], + uiSettings: IUiSettingsClient ): Promise { + const excludedDataTiers = + uiSettings?.get('observability:searchExcludedDataTiers') ?? []; + return new Promise((resolve, reject) => { let esResponse: estypes.SearchResponse | undefined; @@ -100,7 +108,7 @@ async function fetchLogsOverview( index: logParams.index, body: { size: 0, - query: buildLogOverviewQuery(logParams, params), + query: buildLogOverviewQuery(params, excludedDataTiers), aggs: buildLogOverviewAggregations(logParams, params), }, }, @@ -119,14 +127,21 @@ async function fetchLogsOverview( }); } -function buildLogOverviewQuery(logParams: LogParams, params: FetchDataParams) { +function buildLogOverviewQuery(params: FetchDataParams, excludedDataTiers: DataTier[]) { return { - range: { - [TIMESTAMP_FIELD]: { - gt: new Date(params.absoluteTime.start).toISOString(), - lte: new Date(params.absoluteTime.end).toISOString(), - format: 'strict_date_optional_time', - }, + bool: { + must: [ + { + range: { + [TIMESTAMP_FIELD]: { + gt: new Date(params.absoluteTime.start).toISOString(), + lte: new Date(params.absoluteTime.end).toISOString(), + format: 'strict_date_optional_time', + }, + }, + }, + ], + must_not: excludedDataTiers.length ? [{ terms: { _tier: excludedDataTiers } }] : undefined, }, }; } diff --git a/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts b/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts index e2e5cd93a6154..3344865503714 100644 --- a/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts +++ b/x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts @@ -6,6 +6,9 @@ */ import type { KibanaRequest } from '@kbn/core/server'; +import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { InfraPluginRequestHandlerContext } from '../types'; import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter'; @@ -16,7 +19,32 @@ export const createSearchClient = framework: KibanaFramework, request?: KibanaRequest ) => - ( + async ( opts: CallWithRequestParams - ): Promise> => - framework.callWithRequest(requestContext, 'search', opts, request); + ): Promise> => { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get(searchExcludedDataTiers); + + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + return framework.callWithRequest( + requestContext, + 'search', + { + ...opts, + body: { + ...(opts.body ?? {}), + query: { + bool: { + ...(opts.body?.query ? { must: [opts.body?.query] } : {}), + filter: excludedQuery, + }, + }, + }, + }, + request + ); + }; diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts index 4290db429aead..7fda86a9baa6e 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts @@ -186,7 +186,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => async (requestContext, request, response) => { const { sourceId } = request.params; - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const soClient = (await requestContext.core).savedObjects.client; const source = await libs.sources.getSourceConfiguration(soClient, sourceId); diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts index 0e33c8733b59c..c5b25a24601a9 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts @@ -23,7 +23,7 @@ export const initOverviewRoute = (libs: InfraBackendLibs) => { }, async (requestContext, request, response) => { const options = request.body; - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const soClient = (await requestContext.core).savedObjects.client; const source = await libs.sources.getSourceConfiguration(soClient, options.sourceId); diff --git a/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts b/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts index a6f7f998572b4..980f591825d48 100644 --- a/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts +++ b/x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts @@ -57,7 +57,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => { ); UsageCollector.countNode(snapshotRequest.nodeType); - const client = createSearchClient(requestContext, framework, request); + const client = await createSearchClient(requestContext, framework, request); const snapshotResponse = await getNodes( client, diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts b/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts index 2fa4f887e9983..3a2e466a29463 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts +++ b/x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts @@ -6,12 +6,36 @@ */ import type { RequestHandlerContext } from '@kbn/core/server'; -import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; +import type { DataTier } from '@kbn/observability-shared-plugin/common'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter'; +import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework'; export const createSearchClient = (requestContext: RequestHandlerContext, framework: KibanaFramework) => - ( + async ( opts: CallWithRequestParams - ): Promise> => - framework.callWithRequest(requestContext, 'search', opts); + ): Promise> => { + const { uiSettings } = await requestContext.core; + + const excludedDataTiers = await uiSettings.client.get( + 'observability:searchExcludedDataTiers' + ); + + const excludedQuery = excludedDataTiers.length + ? excludeTiersQuery(excludedDataTiers) + : undefined; + + return framework.callWithRequest(requestContext, 'search', { + ...opts, + body: { + ...(opts.body ?? {}), + query: { + bool: { + ...(opts.body?.query ? { must: [opts.body?.query] } : {}), + filter: excludedQuery, + }, + }, + }, + }); + }; diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts b/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts index 44392684d6566..1d2de6b65667b 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts +++ b/x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts @@ -41,7 +41,7 @@ export const initMetricExplorerRoute = (framework: KibanaFramework) => { ); } - const client = createSearchClient(requestContext, framework); + const client = await createSearchClient(requestContext, framework); const interval = await findIntervalForMetrics(client, options); const optionsWithInterval = options.forceInterval diff --git a/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json b/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json index 2d995041b8661..0fe8d1e15a373 100644 --- a/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json +++ b/x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json @@ -35,6 +35,7 @@ "@kbn/lens-embeddable-utils", "@kbn/react-kibana-context-render", "@kbn/react-kibana-context-theme", - "@kbn/router-utils" + "@kbn/router-utils", + "@kbn/observability-utils-common" ] }