[8.19] [Infra] Fix legacy Infra search client to respect the frozen tier setting (#246438)

x-pack/platform/plugins/shared/logs_shared/public/services/log_views/exclude_tiers_query.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+
+export function excludeTiersQuery(
+  excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'>
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: excludedDataTiers,
+            },
+          },
+        ],
+      },
+    },
+  ];
+}

x-pack/platform/plugins/shared/logs_shared/public/services/log_views/log_views_client.ts

@@ -6,7 +6,7 @@
  */
 
 import * as rt from 'io-ts';
-import { HttpStart } from '@kbn/core/public';
+import type { HttpStart, IUiSettingsClient } from '@kbn/core/public';
 import type { ISearchGeneric } from '@kbn/search-types';
 import { DataViewsContract } from '@kbn/data-views-plugin/public';
 import { lastValueFrom } from 'rxjs';
@@ -28,7 +28,8 @@ import {
   isNoSuchRemoteClusterError,
 } from '../../../common/log_views';
 import { decodeOrThrow } from '../../../common/runtime_types';
-import { ILogViewsClient } from './types';
+import type { ILogViewsClient } from './types';
+import { excludeTiersQuery } from './exclude_tiers_query';
 
 export class LogViewsClient implements ILogViewsClient {
   constructor(
@@ -69,8 +70,16 @@ export class LogViewsClient implements ILogViewsClient {
     return resolvedLogView;
   }
 
-  public async getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise<LogViewStatus> {
-    return await lastValueFrom(
+  public async getResolvedLogViewStatus(
+    resolvedLogView: ResolvedLogView,
+    uiSettings?: IUiSettingsClient
+  ): Promise<LogViewStatus> {
+    const excludedDataTiers = uiSettings?.get('observability:searchExcludedDataTiers') ?? [];
+    const excludedQuery = excludedDataTiers.length
+      ? excludeTiersQuery(excludedDataTiers)
+      : undefined;
+
+    const indexStatus = await lastValueFrom(
       this.search({
         params: {
           ignore_unavailable: true,
@@ -79,6 +88,7 @@ export class LogViewsClient implements ILogViewsClient {
           size: 0,
           terminate_after: 1,
           track_total_hits: 1,
+          query: excludedQuery ? { bool: { filter: excludedQuery } } : undefined,
         },
       })
     ).then(
@@ -119,6 +129,8 @@ export class LogViewsClient implements ILogViewsClient {
         );
       }
     );
+
+    return indexStatus;
   }
 
   public async putLogView(

x-pack/platform/plugins/shared/logs_shared/public/services/log_views/types.ts

@@ -5,11 +5,12 @@
  * 2.0.
  */
 
-import { HttpStart } from '@kbn/core/public';
-import { ISearchStart } from '@kbn/data-plugin/public';
-import { DataViewsContract } from '@kbn/data-views-plugin/public';
-import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
-import {
+import type { HttpStart } from '@kbn/core/public';
+import type { ISearchStart } from '@kbn/data-plugin/public';
+import type { DataViewsContract } from '@kbn/data-views-plugin/public';
+import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
+import type { IUiSettingsClient } from '@kbn/core/public';
+import type {
   LogView,
   LogViewAttributes,
   LogViewReference,
@@ -35,7 +36,10 @@ export interface LogViewsServiceStartDeps {
 
 export interface ILogViewsClient {
   getLogView(logViewReference: LogViewReference): Promise<LogView>;
-  getResolvedLogViewStatus(resolvedLogView: ResolvedLogView): Promise<LogViewStatus>;
+  getResolvedLogViewStatus(
+    resolvedLogView: ResolvedLogView,
+    uiSettings?: IUiSettingsClient
+  ): Promise<LogViewStatus>;
   getResolvedLogView(logViewReference: LogViewReference): Promise<ResolvedLogView>;
   putLogView(
     logViewReference: LogViewReference,

x-pack/solutions/observability/plugins/infra/public/utils/logs_overview_fetchers.ts

@@ -12,6 +12,8 @@ import type {
   LogsFetchDataResponse,
 } from '@kbn/observability-plugin/public';
 import { DEFAULT_LOG_VIEW, getLogsLocatorsFromUrlService } from '@kbn/logs-shared-plugin/common';
+import type { IUiSettingsClient } from '@kbn/core/public';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
 import { TIMESTAMP_FIELD } from '../../common/constants';
 import type { InfraClientStartDeps, InfraClientStartServicesAccessor } from '../types';
 
@@ -37,10 +39,11 @@ type StatsAndSeries = Pick<LogsFetchDataResponse, 'stats' | 'series'>;
 
 export function getLogsHasDataFetcher(getStartServices: InfraClientStartServicesAccessor) {
   return async () => {
-    const [, { logsShared }] = await getStartServices();
+    const [coreStart, { logsShared }] = await getStartServices();
     const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW);
     const logViewStatus = await logsShared.logViews.client.getResolvedLogViewStatus(
-      resolvedLogView
+      resolvedLogView,
+      coreStart.uiSettings
     );
 
     const hasData = logViewStatus.index === 'available';
@@ -57,15 +60,16 @@ export function getLogsOverviewDataFetcher(
   getStartServices: InfraClientStartServicesAccessor
 ): FetchData<LogsFetchDataResponse> {
   return async (params) => {
-    const [, { data, logsShared, share }] = await getStartServices();
+    const [coreStart, { data, logsShared, share }] = await getStartServices();
     const resolvedLogView = await logsShared.logViews.client.getResolvedLogView(DEFAULT_LOG_VIEW);
 
     const { stats, series } = await fetchLogsOverview(
       {
         index: resolvedLogView.indices,
       },
       params,
-      data
+      data,
+      coreStart.uiSettings
     );
     const { logsLocator } = getLogsLocatorsFromUrlService(share.url);
     const timeSpanInMinutes = (params.absoluteTime.end - params.absoluteTime.start) / (1000 * 60);
@@ -89,8 +93,12 @@ export function getLogsOverviewDataFetcher(
 async function fetchLogsOverview(
   logParams: LogParams,
   params: FetchDataParams,
-  dataPlugin: InfraClientStartDeps['data']
+  dataPlugin: InfraClientStartDeps['data'],
+  uiSettings: IUiSettingsClient
 ): Promise<StatsAndSeries> {
+  const excludedDataTiers =
+    uiSettings?.get<DataTier[]>('observability:searchExcludedDataTiers') ?? [];
+
   return new Promise((resolve, reject) => {
     let esResponse: estypes.SearchResponse<any> | undefined;
 
@@ -100,7 +108,7 @@ async function fetchLogsOverview(
           index: logParams.index,
           body: {
             size: 0,
-            query: buildLogOverviewQuery(logParams, params),
+            query: buildLogOverviewQuery(params, excludedDataTiers),
             aggs: buildLogOverviewAggregations(logParams, params),
           },
         },
@@ -119,14 +127,21 @@ async function fetchLogsOverview(
   });
 }
 
-function buildLogOverviewQuery(logParams: LogParams, params: FetchDataParams) {
+function buildLogOverviewQuery(params: FetchDataParams, excludedDataTiers: DataTier[]) {
   return {
-    range: {
-      [TIMESTAMP_FIELD]: {
-        gt: new Date(params.absoluteTime.start).toISOString(),
-        lte: new Date(params.absoluteTime.end).toISOString(),
-        format: 'strict_date_optional_time',
-      },
+    bool: {
+      must: [
+        {
+          range: {
+            [TIMESTAMP_FIELD]: {
+              gt: new Date(params.absoluteTime.start).toISOString(),
+              lte: new Date(params.absoluteTime.end).toISOString(),
+              format: 'strict_date_optional_time',
+            },
+          },
+        },
+      ],
+      must_not: excludedDataTiers.length ? [{ terms: { _tier: excludedDataTiers } }] : undefined,
     },
   };
 }

x-pack/solutions/observability/plugins/infra/server/lib/create_search_client.ts

@@ -6,6 +6,9 @@
  */
 
 import type { KibanaRequest } from '@kbn/core/server';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
 import type { InfraPluginRequestHandlerContext } from '../types';
 import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework';
 import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter';
@@ -16,7 +19,32 @@ export const createSearchClient =
     framework: KibanaFramework,
     request?: KibanaRequest
   ) =>
-  <Hit = {}, Aggregation = undefined>(
+  async <Hit = {}, Aggregation = undefined>(
     opts: CallWithRequestParams
-  ): Promise<InfraDatabaseSearchResponse<Hit, Aggregation>> =>
-    framework.callWithRequest(requestContext, 'search', opts, request);
+  ): Promise<InfraDatabaseSearchResponse<Hit, Aggregation>> => {
+    const { uiSettings } = await requestContext.core;
+
+    const excludedDataTiers = await uiSettings.client.get<DataTier[]>(searchExcludedDataTiers);
+
+    const excludedQuery = excludedDataTiers.length
+      ? excludeTiersQuery(excludedDataTiers)
+      : undefined;
+
+    return framework.callWithRequest(
+      requestContext,
+      'search',
+      {
+        ...opts,
+        body: {
+          ...(opts.body ?? {}),
+          query: {
+            bool: {
+              ...(opts.body?.query ? { must: [opts.body?.query] } : {}),
+              filter: excludedQuery,
+            },
+          },
+        },
+      },
+      request
+    );
+  };

x-pack/solutions/observability/plugins/infra/server/routes/metrics_sources/index.ts

@@ -186,7 +186,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) =>
     async (requestContext, request, response) => {
       const { sourceId } = request.params;
 
-      const client = createSearchClient(requestContext, framework);
+      const client = await createSearchClient(requestContext, framework);
       const soClient = (await requestContext.core).savedObjects.client;
       const source = await libs.sources.getSourceConfiguration(soClient, sourceId);
 

x-pack/solutions/observability/plugins/infra/server/routes/overview/index.ts

@@ -23,7 +23,7 @@ export const initOverviewRoute = (libs: InfraBackendLibs) => {
     },
     async (requestContext, request, response) => {
       const options = request.body;
-      const client = createSearchClient(requestContext, framework);
+      const client = await createSearchClient(requestContext, framework);
       const soClient = (await requestContext.core).savedObjects.client;
       const source = await libs.sources.getSourceConfiguration(soClient, options.sourceId);
 

x-pack/solutions/observability/plugins/infra/server/routes/snapshot/index.ts

@@ -57,7 +57,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => {
           );
 
         UsageCollector.countNode(snapshotRequest.nodeType);
-        const client = createSearchClient(requestContext, framework, request);
+        const client = await createSearchClient(requestContext, framework, request);
 
         const snapshotResponse = await getNodes(
           client,

x-pack/solutions/observability/plugins/metrics_data_access/server/lib/create_search_client.ts

@@ -6,12 +6,36 @@
  */
 
 import type { RequestHandlerContext } from '@kbn/core/server';
-import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
 import type { KibanaFramework } from './adapters/framework/kibana_framework_adapter';
+import type { CallWithRequestParams, InfraDatabaseSearchResponse } from './adapters/framework';
 
 export const createSearchClient =
   (requestContext: RequestHandlerContext, framework: KibanaFramework) =>
-  <Hit = {}, Aggregation = undefined>(
+  async <Hit = {}, Aggregation = undefined>(
     opts: CallWithRequestParams
-  ): Promise<InfraDatabaseSearchResponse<Hit, Aggregation>> =>
-    framework.callWithRequest(requestContext, 'search', opts);
+  ): Promise<InfraDatabaseSearchResponse<Hit, Aggregation>> => {
+    const { uiSettings } = await requestContext.core;
+
+    const excludedDataTiers = await uiSettings.client.get<DataTier[]>(
+      'observability:searchExcludedDataTiers'
+    );
+
+    const excludedQuery = excludedDataTiers.length
+      ? excludeTiersQuery(excludedDataTiers)
+      : undefined;
+
+    return framework.callWithRequest(requestContext, 'search', {
+      ...opts,
+      body: {
+        ...(opts.body ?? {}),
+        query: {
+          bool: {
+            ...(opts.body?.query ? { must: [opts.body?.query] } : {}),
+            filter: excludedQuery,
+          },
+        },
+      },
+    });
+  };

x-pack/solutions/observability/plugins/metrics_data_access/server/routes/metrics_explorer/index.ts

@@ -41,7 +41,7 @@ export const initMetricExplorerRoute = (framework: KibanaFramework) => {
           );
         }
 
-        const client = createSearchClient(requestContext, framework);
+        const client = await createSearchClient(requestContext, framework);
         const interval = await findIntervalForMetrics(client, options);
 
         const optionsWithInterval = options.forceInterval

x-pack/solutions/observability/plugins/metrics_data_access/tsconfig.json

@@ -35,6 +35,7 @@
     "@kbn/lens-embeddable-utils",
     "@kbn/react-kibana-context-render",
     "@kbn/react-kibana-context-theme",
-    "@kbn/router-utils"
+    "@kbn/router-utils",
+    "@kbn/observability-utils-common"
   ]
 }