[8.0] [Security Solution] [Sourcerer] [Feature Branch] Update to use Kibana Data Views (#114806)

docs/api/dashboard-api.asciidoc

@@ -1,7 +1,7 @@
 [[dashboard-api]]
 == Import and export dashboard APIs
 
-deprecated::[7.15.0,These experimental APIs have been deprecated in favor of <<saved-objects-api-import>> and <<saved-objects-api-export>>.]
+deprecated::[7.15.0,Both of these APIs have been deprecated in favor of <<saved-objects-api-import>> and <<saved-objects-api-export>>.]
 
 Import and export dashboards with the corresponding saved objects, such as visualizations, saved
 searches, and index patterns.

docs/api/dashboard/export-dashboard.asciidoc

@@ -6,7 +6,7 @@
 
 deprecated::[7.15.0,Use <<saved-objects-api-export>> instead.]
 
-experimental[] Export dashboards and corresponding saved objects.
+Export dashboards and corresponding saved objects.
 
 [[dashboard-api-export-request]]
 ==== Request

docs/api/dashboard/import-dashboard.asciidoc

@@ -6,7 +6,7 @@
 
 deprecated::[7.15.0,Use <<saved-objects-api-import>> instead.]
 
-experimental[] Import dashboards and corresponding saved objects.
+Import dashboards and corresponding saved objects.
 
 [[dashboard-api-import-request]]
 ==== Request

packages/kbn-securitysolution-autocomplete/src/field/index.tsx

@@ -8,7 +8,7 @@
 
 import React, { useCallback, useMemo, useState } from 'react';
 import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
-import { IndexPatternBase, IndexPatternFieldBase } from '@kbn/es-query';
+import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
 
 import {
   getGenericComboBoxProps,
@@ -20,14 +20,14 @@ const AS_PLAIN_TEXT = { asPlainText: true };
 interface OperatorProps {
   fieldInputWidth?: number;
   fieldTypeFilter?: string[];
-  indexPattern: IndexPatternBase | undefined;
+  indexPattern: DataViewBase | undefined;
   isClearable: boolean;
   isDisabled: boolean;
   isLoading: boolean;
   isRequired?: boolean;
-  onChange: (a: IndexPatternFieldBase[]) => void;
+  onChange: (a: DataViewFieldBase[]) => void;
   placeholder: string;
-  selectedField: IndexPatternFieldBase | undefined;
+  selectedField: DataViewFieldBase | undefined;
 }
 
 export const FieldComponent: React.FC<OperatorProps> = ({
@@ -56,7 +56,7 @@ export const FieldComponent: React.FC<OperatorProps> = ({
 
   const handleValuesChange = useCallback(
     (newOptions: EuiComboBoxOptionOption[]): void => {
-      const newValues: IndexPatternFieldBase[] = newOptions.map(
+      const newValues: DataViewFieldBase[] = newOptions.map(
         ({ label }) => availableFields[labels.indexOf(label)]
       );
       onChange(newValues);
@@ -94,13 +94,13 @@ export const FieldComponent: React.FC<OperatorProps> = ({
 FieldComponent.displayName = 'Field';
 
 interface ComboBoxFields {
-  availableFields: IndexPatternFieldBase[];
-  selectedFields: IndexPatternFieldBase[];
+  availableFields: DataViewFieldBase[];
+  selectedFields: DataViewFieldBase[];
 }
 
 const getComboBoxFields = (
-  indexPattern: IndexPatternBase | undefined,
-  selectedField: IndexPatternFieldBase | undefined,
+  indexPattern: DataViewBase | undefined,
+  selectedField: DataViewFieldBase | undefined,
   fieldTypeFilter: string[]
 ): ComboBoxFields => {
   const existingFields = getExistingFields(indexPattern);
@@ -113,29 +113,27 @@ const getComboBoxFields = (
 const getComboBoxProps = (fields: ComboBoxFields): GetGenericComboBoxPropsReturn => {
   const { availableFields, selectedFields } = fields;
 
-  return getGenericComboBoxProps<IndexPatternFieldBase>({
+  return getGenericComboBoxProps<DataViewFieldBase>({
     getLabel: (field) => field.name,
     options: availableFields,
     selectedOptions: selectedFields,
   });
 };
 
-const getExistingFields = (indexPattern: IndexPatternBase | undefined): IndexPatternFieldBase[] => {
+const getExistingFields = (indexPattern: DataViewBase | undefined): DataViewFieldBase[] => {
   return indexPattern != null ? indexPattern.fields : [];
 };
 
-const getSelectedFields = (
-  selectedField: IndexPatternFieldBase | undefined
-): IndexPatternFieldBase[] => {
+const getSelectedFields = (selectedField: DataViewFieldBase | undefined): DataViewFieldBase[] => {
   return selectedField ? [selectedField] : [];
 };
 
 const getAvailableFields = (
-  existingFields: IndexPatternFieldBase[],
-  selectedFields: IndexPatternFieldBase[],
+  existingFields: DataViewFieldBase[],
+  selectedFields: DataViewFieldBase[],
   fieldTypeFilter: string[]
-): IndexPatternFieldBase[] => {
-  const fieldsByName = new Map<string, IndexPatternFieldBase>();
+): DataViewFieldBase[] => {
+  const fieldsByName = new Map<string, DataViewFieldBase>();
 
   existingFields.forEach((f) => fieldsByName.set(f.name, f));
   selectedFields.forEach((f) => fieldsByName.set(f.name, f));

src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -5,7 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
 import { IndexPatternsFetcher } from '.';
 import { ElasticsearchClient } from 'kibana/server';
 import * as indexNotFoundException from './index_not_found_exception.json';
@@ -15,36 +14,36 @@ describe('Index Pattern Fetcher - server', () => {
   let esClient: ElasticsearchClient;
   const emptyResponse = {
     body: {
-      count: 0,
+      indices: [],
     },
   };
   const response = {
     body: {
-      count: 1115,
+      indices: ['b'],
+      fields: [{ name: 'foo' }, { name: 'bar' }, { name: 'baz' }],
     },
   };
   const patternList = ['a', 'b', 'c'];
   beforeEach(() => {
+    jest.clearAllMocks();
     esClient = {
-      count: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
   });
-
   it('Removes pattern without matching indices', async () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(['b', 'c']);
   });
-
   it('Returns all patterns when all match indices', async () => {
     esClient = {
-      count: jest.fn().mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(patternList);
   });
-  it('Removes pattern when "index_not_found_exception" error is thrown', async () => {
+  it('Removes pattern when error is thrown', async () => {
     class ServerError extends Error {
       public body?: Record<string, any>;
       constructor(
@@ -56,9 +55,8 @@ describe('Index Pattern Fetcher - server', () => {
         this.body = errBody;
       }
     }
-
     esClient = {
-      count: jest
+      fieldCaps: jest
         .fn()
         .mockResolvedValueOnce(response)
         .mockRejectedValue(
@@ -69,4 +67,22 @@ describe('Index Pattern Fetcher - server', () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual([patternList[0]]);
   });
+  it('When allowNoIndices is false, run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(4);
+  });
+  it('When allowNoIndices is true, do not run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient, true);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(1);
+  });
 });

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -36,12 +36,10 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
   private elasticsearchClient: ElasticsearchClient;
   private allowNoIndices: boolean;
-
   constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
     this.elasticsearchClient = elasticsearchClient;
     this.allowNoIndices = allowNoIndices;
   }
-
   /**
    *  Get a list of field objects for an index pattern that may contain wildcards
    *
@@ -60,23 +58,22 @@ export class IndexPatternsFetcher {
   }): Promise<FieldDescriptor[]> {
     const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
     const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
+    const allowNoIndices = fieldCapsOptions
+      ? fieldCapsOptions.allow_no_indices
+      : this.allowNoIndices;
     let patternListActive: string[] = patternList;
     // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
-    if (patternList.length > 1) {
+    if (patternList.length > 1 && !allowNoIndices) {
       patternListActive = await this.validatePatternListActive(patternList);
     }
     const fieldCapsResponse = await getFieldCapabilities(
       this.elasticsearchClient,
-      // if none of the patterns are active, pass the original list to get an error
-      patternListActive.length > 0 ? patternListActive : patternList,
+      patternListActive,
       metaFields,
       {
-        allow_no_indices: fieldCapsOptions
-          ? fieldCapsOptions.allow_no_indices
-          : this.allowNoIndices,
+        allow_no_indices: allowNoIndices,
       }
     );
-
     if (type === 'rollup' && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
       const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -87,13 +84,11 @@ export class IndexPatternsFetcher {
         ).body
       )[rollupIndex].aggs;
       const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
-
       // Keep meta fields
       metaFields!.forEach(
         (field: string) =>
           fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
       );
-
       return mergeCapabilitiesWithFields(
         rollupIndexCapabilities,
         fieldCapsResponseObj,
@@ -137,23 +132,20 @@ export class IndexPatternsFetcher {
   async validatePatternListActive(patternList: string[]) {
     const result = await Promise.all(
       patternList
-        .map((pattern) =>
-          this.elasticsearchClient.count({
-            index: pattern,
-          })
-        )
-        .map((p) =>
-          p.catch((e) => {
-            if (e.body.error.type === 'index_not_found_exception') {
-              return { body: { count: 0 } };
-            }
-            throw e;
-          })
-        )
+        .map(async (index) => {
+          const searchResponse = await this.elasticsearchClient.fieldCaps({
+            index,
+            fields: '_id',
+            ignore_unavailable: true,
+            allow_no_indices: false,
+          });
+          return searchResponse.body.indices.length > 0;
+        })
+        .map((p) => p.catch(() => false))
     );
     return result.reduce(
-      (acc: string[], { body: { count } }, patternListIndex) =>
-        count > 0 ? [...acc, patternList[patternListIndex]] : acc,
+      (acc: string[], isValid, patternListIndex) =>
+        isValid ? [...acc, patternList[patternListIndex]] : acc,
       []
     );
   }

x-pack/plugins/observability/public/pages/alerts/alerts_table_t_grid.tsx

@@ -385,6 +385,8 @@ export function AlertsTableTGrid(props: AlertsTableTGridProps) {
       },
       renderCellValue: getRenderCellValue({ setFlyoutAlert }),
       rowRenderers: NO_ROW_RENDER,
+      // TODO: implement Kibana data view runtime fields in observability
+      runtimeMappings: {},
       start: rangeFrom,
       setRefetch,
       sort: [

x-pack/plugins/security_solution/common/constants.ts

@@ -9,8 +9,16 @@ import type { TransformConfigSchema } from './transforms/types';
 import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
 import { METADATA_TRANSFORMS_PATTERN } from './endpoint/constants';
 
+/**
+ * as const
+ *
+ * The const assertion ensures that type widening does not occur
+ * https://mariusschulz.com/blog/literal-type-widening-in-typescript
+ * Please follow this convention when adding to this file
+ */
+
 export const APP_ID = 'securitySolution' as const;
-export const APP_UI_ID = 'securitySolutionUI';
+export const APP_UI_ID = 'securitySolutionUI' as const;
 export const CASES_FEATURE_ID = 'securitySolutionCases' as const;
 export const SERVER_APP_ID = 'siem' as const;
 export const APP_NAME = 'Security' as const;
@@ -24,6 +32,8 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const;
 export const DEFAULT_DARK_MODE = 'theme:darkMode' as const;
 export const DEFAULT_INDEX_KEY = 'securitySolution:defaultIndex' as const;
 export const DEFAULT_NUMBER_FORMAT = 'format:number:defaultPattern' as const;
+export const DEFAULT_DATA_VIEW_ID = 'security-solution' as const;
+export const DEFAULT_TIME_FIELD = '@timestamp' as const;
 export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults' as const;
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults' as const;
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const;
@@ -49,7 +59,6 @@ export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges' as const
 export const DEFAULT_TRANSFORMS = 'securitySolution:transforms' as const;
 export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled' as const;
 export const GLOBAL_HEADER_HEIGHT = 96 as const; // px
-export const GLOBAL_HEADER_HEIGHT_WITH_GLOBAL_BANNER = 128 as const; // px
 export const FILTERS_GLOBAL_HEIGHT = 109 as const; // px
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled' as const;
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51' as const;
@@ -266,6 +275,7 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged` as const;
 
 export const NOTE_URL = '/api/note' as const;
 export const PINNED_EVENT_URL = '/api/pinned_event' as const;
+export const SOURCERER_API_URL = '/api/sourcerer' as const;
 
 /**
  * Default signals index key for kibana.dev.yml
@@ -349,7 +359,7 @@ export const ELASTIC_NAME = 'estc' as const;
 
 export const METADATA_TRANSFORM_STATS_URL = `/api/transform/transforms/${METADATA_TRANSFORMS_PATTERN}/_stats`;
 
-export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_';
+export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_' as const;
 
 export const TRANSFORM_STATES = {
   ABORTING: 'aborting',

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts

@@ -11,7 +11,7 @@ import type {
   CreateExceptionListItemSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
 import { buildExceptionFilter } from '@kbn/securitysolution-list-utils';
-import { Filter, EsQueryConfig, IndexPatternBase, buildEsQuery } from '@kbn/es-query';
+import { Filter, EsQueryConfig, DataViewBase, buildEsQuery } from '@kbn/es-query';
 
 import { ESBoolQuery } from '../typed_json';
 import { Query, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
@@ -24,7 +24,7 @@ export const getQueryFilter = (
   lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>,
   excludeExceptions: boolean = true
 ): ESBoolQuery => {
-  const indexPattern: IndexPatternBase = {
+  const indexPattern: DataViewBase = {
     fields: [],
     title: index.join(),
   };