[One Discover] [Logs UX]Add support to disable highlighting for certain filters we deem internal

src/platform/packages/shared/kbn-saved-search-component/src/components/saved_search.tsx

@@ -34,6 +34,7 @@ export const SavedSearchComponent: React.FC<SavedSearchComponentProps> = (props)
     timeRange,
     query,
     filters,
+    nonHighlightingFilters,
     index,
     timestampField,
     columns,
@@ -64,6 +65,7 @@ export const SavedSearchComponent: React.FC<SavedSearchComponentProps> = (props)
           searchSource.setField('index', dataView);
           searchSource.setField('query', query);
           searchSource.setField('filter', filters);
+          searchSource.setField('nonHighlightingFilters', nonHighlightingFilters);
           const { searchSourceJSON, references } = searchSource.serialize();
           // By-value saved object structure
           const savedSearch: SavedSearch = {
@@ -106,6 +108,7 @@ export const SavedSearchComponent: React.FC<SavedSearchComponentProps> = (props)
     dataViews,
     documentViewerEnabled,
     filters,
+    nonHighlightingFilters,
     filtersEnabled,
     index,
     query,

src/platform/packages/shared/kbn-saved-search-component/src/types.ts

@@ -26,6 +26,12 @@ export interface SavedSearchComponentProps {
   timeRange?: TimeRange;
   query?: Query;
   filters?: Filter[];
+  /**
+   * Filters that should not trigger highlighting.
+   * These filters will be included in the search query for document retrieval,
+   * but excluded from the highlight_query parameter in Elasticsearch.
+   */
+  nonHighlightingFilters?: Filter[];
   timestampField?: string;
   columns?: string[];
   height?: CSSProperties['height'];

src/platform/plugins/shared/data/common/search/search_source/search_source.ts

@@ -645,10 +645,12 @@ export class SearchSource {
       case 'filter':
         return addToRoot(
           'filters',
-          (typeof data.filters === 'function' ? data.filters() : data.filters || []).concat(val)
+          (typeof data.filters === 'function' ? data.filters() : data.filters ?? []).concat(val)
         );
+      case 'nonHighlightingFilters':
+        return addToRoot('nonHighlightingFilters', (data.nonHighlightingFilters ?? []).concat(val));
       case 'query':
-        return addToRoot(key, (data.query || []).concat(val));
+        return addToRoot(key, (data.query ?? []).concat(val));
       case 'fields':
         // This will pass the passed in parameters to the new fields API.
         // Also if will only return scripted fields that are part of the specified
@@ -791,6 +793,7 @@ export class SearchSource {
     const bodyParams = omit(searchRequest, [
       'index',
       'filters',
+      'nonHighlightingFilters',
       'highlightAll',
       'fieldsFromSource',
       'body',
@@ -879,21 +882,51 @@ export class SearchSource {
       });
     }
 
+    // Evaluate filters if they are functions
+    const filters =
+      typeof searchRequest.filters === 'function' ? searchRequest.filters() : searchRequest.filters;
+
+    const nonHighlightingFilters = searchRequest.nonHighlightingFilters ?? [];
+
+    // Merge filters and nonHighlightingFilters for the main query
+    const allFilters = [
+      ...(Array.isArray(filters) ? filters : filters ? [filters] : []),
+      ...nonHighlightingFilters,
+    ];
+
     const builtQuery = this.getBuiltEsQuery({
       index: searchRequest.index,
       query: searchRequest.query,
-      filters: searchRequest.filters,
+      filters: allFilters,
       getConfig,
       sort: body.sort,
     });
 
+    // Build highlight query using only filters (not nonHighlightingFilters)
+    const filterArray = Array.isArray(filters) ? filters : filters ? [filters] : [];
+    const highlightQuery =
+      searchRequest.highlightAll && filterArray.length > 0
+        ? this.getBuiltEsQuery({
+            index: searchRequest.index,
+            query: searchRequest.query,
+            filters: filterArray,
+            getConfig,
+            sort: body.sort,
+          })
+        : undefined;
+
     const bodyToReturn = {
       ...body,
       pit: searchRequest.pit,
       query: builtQuery,
       highlight:
         searchRequest.highlightAll && builtQuery
-          ? getHighlightRequest(getConfig(UI_SETTINGS.DOC_HIGHLIGHT))
+          ? highlightQuery
+            ? {
+                ...getHighlightRequest(getConfig(UI_SETTINGS.DOC_HIGHLIGHT)),
+                highlight_query: highlightQuery,
+              }
+            : getHighlightRequest(getConfig(UI_SETTINGS.DOC_HIGHLIGHT))
           : undefined,
       // remove _source, since everything's coming from fields API, scripted, or stored fields
       _source: fieldListProvided && !sourceFieldsProvided ? false : body._source,
@@ -914,7 +947,7 @@ export class SearchSource {
     };
 
     return omitByIsNil({
-      ...omit(searchRequest, ['query', 'filters', 'fieldsFromSource']),
+      ...omit(searchRequest, ['query', 'filters', 'nonHighlightingFilters', 'fieldsFromSource']),
       body: omitByIsNil(bodyToReturn),
       indexType: this.getIndexType(searchRequest.index),
       highlightAll:
@@ -1091,6 +1124,7 @@ export class SearchSource {
   public getSerializedFields(recurse = false): SerializedSearchSourceFields {
     const {
       filter: originalFilters,
+      nonHighlightingFilters: originalNonHighlightingFilters,
       aggs: searchSourceAggs,
       parent,
       size: _size, // omit it
@@ -1115,6 +1149,12 @@ export class SearchSource {
         filter: filters,
       };
     }
+    if (originalNonHighlightingFilters) {
+      serializedSearchSourceFields = {
+        ...serializedSearchSourceFields,
+        nonHighlightingFilters: originalNonHighlightingFilters,
+      };
+    }
     if (searchSourceAggs) {
       let aggs = searchSourceAggs;
       if (typeof aggs === 'function') {

src/platform/plugins/shared/data/common/search/search_source/types.ts

@@ -80,6 +80,13 @@ export interface SearchSourceFields {
    * {@link Filter}
    */
   filter?: Filter[] | Filter | (() => Filter[] | Filter | undefined);
+  /**
+   * Filters that should not trigger highlighting.
+   * These filters will be included in the search query for document retrieval,
+   * but excluded from the highlight_query parameter in Elasticsearch.
+   * {@link Filter}
+   */
+  nonHighlightingFilters?: Filter[];
   /**
    * {@link EsQuerySortValue}
    */
@@ -131,6 +138,13 @@ export type SerializedSearchSourceFields = {
    * {@link Filter}
    */
   filter?: Filter[];
+  /**
+   * Filters that should not trigger highlighting.
+   * These filters will be included in the search query for document retrieval,
+   * but excluded from the highlight_query parameter in Elasticsearch.
+   * {@link Filter}
+   */
+  nonHighlightingFilters?: Filter[];
   /**
    * {@link EsQuerySortValue}
    */

src/platform/plugins/shared/discover_shared/public/services/discover_features/types.ts

@@ -84,7 +84,8 @@ export interface ObservabilityCreateSLOFeature {
 export interface ObservabilityLogEventsFeature {
   id: 'observability-log-events';
   render: (props: {
-    query: Query;
+    query?: Query;
+    nonHighlightingQuery?: Query;
     timeRange: TimeRange;
     index: string;
     displayOptions?: {

src/platform/plugins/shared/unified_doc_viewer/public/components/observability/traces/components/trace_context_log_events/index.tsx

@@ -87,7 +87,11 @@ export function TraceContextLogEvents({
       }
     >
       <div tabIndex={0} className="eui-yScrollWithShadows" style={{ maxHeight: '400px' }}>
-        <LogEventsComponent query={query} timeRange={savedSearchTimeRange} index={indexes.logs} />
+        <LogEventsComponent
+          nonHighlightingQuery={query}
+          timeRange={savedSearchTimeRange}
+          index={indexes.logs}
+        />
       </div>
     </ContentFrameworkSection>
   );

x-pack/platform/packages/shared/logs-overview/src/components/log_categories/log_categories.tsx

@@ -40,6 +40,7 @@ import { LogCategoriesResultContent } from './log_categories_result_content';
 export type LogCategoriesProps = LogCategoriesContentProps & {
   dependencies: LogCategoriesDependencies;
   documentFilters: QueryDslQueryContainer[];
+  nonHighlightingFilters?: QueryDslQueryContainer[];
   logsSource: ResolvedIndexNameLogsSourceConfiguration;
   // The time range could be made optional if we want to support an internal
   // time range picker

x-pack/platform/packages/shared/logs-overview/src/components/log_events/log_events.tsx

@@ -32,13 +32,18 @@ export const LogEvents = React.memo<LogEventsProps>(
   ({
     dependencies,
     documentFilters,
+    nonHighlightingFilters,
     logsSource,
     timeRange,
     grouping,
     groupingCapabilities,
     height,
     onChangeGrouping,
   }) => {
+    const allFilters = React.useMemo(
+      () => [...(documentFilters || []), ...(nonHighlightingFilters || [])],
+      [documentFilters, nonHighlightingFilters]
+    );
     return (
       <EuiFlexGroup
         direction="column"
@@ -49,7 +54,7 @@ export const LogEvents = React.memo<LogEventsProps>(
         <EuiFlexItem grow={false}>
           <LogEventsControlBar
             dependencies={dependencies}
-            documentFilters={documentFilters}
+            documentFilters={allFilters}
             logsSource={logsSource}
             timeRange={timeRange}
             grouping={grouping}
@@ -61,6 +66,7 @@ export const LogEvents = React.memo<LogEventsProps>(
           <LogEventsResultContent
             dependencies={dependencies}
             documentFilters={documentFilters}
+            nonHighlightingFilters={nonHighlightingFilters}
             logsSource={logsSource}
             timeRange={timeRange}
           />

x-pack/platform/packages/shared/logs-overview/src/components/log_events/log_events_result_content.tsx

@@ -17,6 +17,7 @@ import type { ResolvedIndexNameLogsSourceConfiguration } from '../../utils/logs_
 export interface LogEventsResultContentProps {
   dependencies: LogEventsResultContentDependencies;
   documentFilters: QueryDslQueryContainer[];
+  nonHighlightingFilters?: QueryDslQueryContainer[];
   logsSource: ResolvedIndexNameLogsSourceConfiguration;
   timeRange: {
     start: string;
@@ -31,7 +32,7 @@ export interface LogEventsResultContentDependencies {
 }
 
 export const LogEventsResultContent = React.memo<LogEventsResultContentProps>(
-  ({ dependencies, documentFilters, logsSource, timeRange }) => {
+  ({ dependencies, documentFilters, nonHighlightingFilters = [], logsSource, timeRange }) => {
     const savedSearchDependencies = React.useMemo(
       () => ({
         embeddable: dependencies.embeddable,
@@ -64,13 +65,29 @@ export const LogEventsResultContent = React.memo<LogEventsResultContentProps>(
       [documentFilters, logsSource.indexName]
     );
 
+    const savedSearchNonHighlightingFilters = React.useMemo(
+      () =>
+        nonHighlightingFilters.map((filter) =>
+          buildCustomFilter(
+            logsSource.indexName,
+            filter,
+            false,
+            false,
+            'Context Filters',
+            FilterStateStore.APP_STATE
+          )
+        ),
+      [nonHighlightingFilters, logsSource.indexName]
+    );
+
     return (
       <LazySavedSearchComponent
         dependencies={savedSearchDependencies}
         index={logsSource.indexName}
         timestampField={logsSource.timestampField}
         timeRange={savedSearchTimeRange}
         filters={savedSearchFilters}
+        nonHighlightingFilters={savedSearchNonHighlightingFilters}
         displayOptions={{
           solutionNavIdOverride: 'oblt',
           enableDocumentViewer: true,

x-pack/platform/packages/shared/logs-overview/src/components/logs_overview/logs_overview.test.tsx

@@ -119,6 +119,7 @@ describe('LogsOverview', () => {
         },
         "filters": Array [],
         "index": "logs",
+        "nonHighlightingFilters": Array [],
         "timeRange": Object {
           "from": "2023-01-01T00:00:00Z",
           "to": "2023-01-02T00:00:00Z",
@@ -160,6 +161,7 @@ describe('LogsOverview', () => {
         },
         "filters": Array [],
         "index": "logs-custom.indexname-*",
+        "nonHighlightingFilters": Array [],
         "timeRange": Object {
           "from": "2023-01-01T00:00:00Z",
           "to": "2023-01-02T00:00:00Z",
@@ -205,6 +207,7 @@ describe('LogsOverview', () => {
         },
         "filters": Array [],
         "index": "logs-custom.dataview-*",
+        "nonHighlightingFilters": Array [],
         "timeRange": Object {
           "from": "2023-01-01T00:00:00Z",
           "to": "2023-01-02T00:00:00Z",

x-pack/platform/packages/shared/logs-overview/src/components/logs_overview/logs_overview.tsx

@@ -27,7 +27,8 @@ import { LogsOverviewStateContext, logsOverviewStateMachine } from './logs_overv
 
 export type LogsOverviewProps = Pick<LogsOverviewContentProps, 'height' | 'timeRange'> & {
   dependencies: LogsOverviewDependencies;
-  documentFilters?: QueryDslQueryContainer[] | undefined;
+  documentFilters?: QueryDslQueryContainer[];
+  nonHighlightingFilters?: QueryDslQueryContainer[];
   featureFlags?: LogsOverviewFeatureFlags | undefined;
   logsSource?: LogsSourceConfiguration | undefined;
 };
@@ -42,6 +43,7 @@ export const LogsOverview: React.FC<LogsOverviewProps> = React.memo(
   ({
     dependencies,
     documentFilters = defaultDocumentFilters,
+    nonHighlightingFilters = defaultNonHighlightingFilters,
     featureFlags = defaultFeatureFlags,
     height,
     logsSource = defaultLogsSource,
@@ -71,6 +73,7 @@ export const LogsOverview: React.FC<LogsOverviewProps> = React.memo(
         <LogsOverviewContent
           dependencies={dependencies}
           documentFilters={documentFilters}
+          nonHighlightingFilters={nonHighlightingFilters}
           height={height}
           timeRange={timeRange}
         />
@@ -81,16 +84,16 @@ export const LogsOverview: React.FC<LogsOverviewProps> = React.memo(
 
 export type LogsOverviewContentProps = Pick<
   LogCategoriesProps,
-  'height' | 'timeRange' | 'documentFilters'
+  'height' | 'timeRange' | 'documentFilters' | 'nonHighlightingFilters'
 > &
-  Pick<LogEventsProps, 'height' | 'timeRange' | 'documentFilters'> & {
+  Pick<LogEventsProps, 'height' | 'timeRange' | 'documentFilters' | 'nonHighlightingFilters'> & {
     dependencies: LogsOverviewDependencies;
   };
 
 export type LogsOverviewContentDependencies = LogCategoriesDependencies & LogEventsDependencies;
 
 export const LogsOverviewContent = React.memo<LogsOverviewContentProps>(
-  ({ dependencies, documentFilters, height, timeRange }) => {
+  ({ dependencies, documentFilters, nonHighlightingFilters, height, timeRange }) => {
     const logsOverviewStateActorRef = LogsOverviewStateContext.useActorRef();
 
     const logsOverviewState = LogsOverviewStateContext.useSelector(identity);
@@ -123,6 +126,11 @@ export const LogsOverviewContent = React.memo<LogsOverviewContentProps>(
       [logsOverviewStateActorRef]
     );
 
+    const allFiltersForCategories = React.useMemo(
+      () => [...(documentFilters || []), ...(nonHighlightingFilters || [])],
+      [documentFilters, nonHighlightingFilters]
+    );
+
     if (logsOverviewState.matches('initializing')) {
       return <LogsOverviewLoadingContent />;
     } else if (logsOverviewState.matches('failedToInitialize')) {
@@ -136,7 +144,7 @@ export const LogsOverviewContent = React.memo<LogsOverviewContentProps>(
         return (
           <LogCategories
             dependencies={dependencies}
-            documentFilters={documentFilters}
+            documentFilters={allFiltersForCategories}
             logsSource={logsOverviewState.context.logsSource.value}
             timeRange={timeRange}
             grouping={grouping}
@@ -150,6 +158,7 @@ export const LogsOverviewContent = React.memo<LogsOverviewContentProps>(
           <LogEvents
             dependencies={dependencies}
             documentFilters={documentFilters}
+            nonHighlightingFilters={nonHighlightingFilters}
             logsSource={logsOverviewState.context.logsSource.value}
             timeRange={timeRange}
             grouping={grouping}
@@ -165,6 +174,8 @@ export const LogsOverviewContent = React.memo<LogsOverviewContentProps>(
 
 const defaultDocumentFilters: QueryDslQueryContainer[] = [];
 
+const defaultNonHighlightingFilters: QueryDslQueryContainer[] = [];
+
 const defaultFeatureFlags: LogsOverviewFeatureFlags = {
   isPatternsEnabled: true,
 };