[Security Solution][Detections] - Rule creation query preview (#78985)

### Summary

This PR introduces a preview histogram feature inside the rule creation workflow, that gives users insight to how effective the rule is for triggering alerts the user would like to see.

Author: yctercero

x-pack/plugins/security_solution/common/detection_engine/types.ts

@@ -8,3 +8,48 @@ import { AlertAction } from '../../../alerts/common';
 export type RuleAlertAction = Omit<AlertAction, 'actionTypeId'> & {
   action_type_id: string;
 };
+
+export type SearchTypes =
+  | string
+  | string[]
+  | number
+  | number[]
+  | boolean
+  | boolean[]
+  | object
+  | object[]
+  | undefined;
+
+export interface Explanation {
+  value: number;
+  description: string;
+  details: Explanation[];
+}
+
+export interface TotalValue {
+  value: number;
+  relation: string;
+}
+
+export interface BaseHit<T> {
+  _index: string;
+  _id: string;
+  _source: T;
+}
+
+export interface EqlSequence<T> {
+  join_keys: SearchTypes[];
+  events: Array<BaseHit<T>>;
+}
+
+export interface EqlSearchResponse<T> {
+  is_partial: boolean;
+  is_running: boolean;
+  took: number;
+  timed_out: boolean;
+  hits: {
+    total: TotalValue;
+    sequences?: Array<EqlSequence<T>>;
+    events?: Array<BaseHit<T>>;
+  };
+}

x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts

@@ -35,6 +35,7 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions {
   timerange: TimerangeInput;
   histogramType: MatrixHistogramType;
   stackByField: string;
+  threshold?: { field: string | undefined; value: number } | undefined;
   inspect?: Maybe<Inspect>;
 }
 

x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx

@@ -51,11 +51,13 @@ const checkIfAnyValidSeriesExist = (
 export const BarChartBaseComponent = ({
   data,
   forceHiddenLegend = false,
+  yAxisTitle,
   ...chartConfigs
 }: {
   data: ChartSeriesData[];
   width: string | null | undefined;
   height: string | null | undefined;
+  yAxisTitle?: string | undefined;
   configs?: ChartSeriesConfigs | undefined;
   forceHiddenLegend?: boolean;
 }) => {
@@ -115,6 +117,7 @@ export const BarChartBaseComponent = ({
           },
         }}
         tickFormat={yTickFormatter}
+        title={yAxisTitle}
       />
     </Chart>
   ) : null;
@@ -158,6 +161,7 @@ export const BarChartComponent: React.FC<BarChartComponentProps> = ({
     [barChart, stackByField, timelineId]
   );
 
+  const yAxisTitle = get('yAxisTitle', configs);
   const customHeight = get('customHeight', configs);
   const customWidth = get('customWidth', configs);
   const chartHeight = getChartHeight(customHeight, height);
@@ -170,6 +174,7 @@ export const BarChartComponent: React.FC<BarChartComponentProps> = ({
           <BarChartBase
             configs={configs}
             data={barChart}
+            yAxisTitle={yAxisTitle}
             forceHiddenLegend={stackByField != null}
             height={chartHeight}
             width={chartHeight}

x-pack/plugins/security_solution/public/common/components/charts/common.tsx

@@ -43,11 +43,14 @@ export interface ChartSeriesConfigs {
   series?: {
     xScaleType?: ScaleType | undefined;
     yScaleType?: ScaleType | undefined;
+    stackAccessors?: string[] | undefined;
   };
   axis?: {
     xTickFormatter?: TickFormatter | undefined;
     yTickFormatter?: TickFormatter | undefined;
+    tickSize?: number | undefined;
   };
+  yAxisTitle?: string | undefined;
   settings?: Partial<SettingsSpecProps>;
 }
 

x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx

@@ -34,6 +34,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps &
     defaultStackByOption: MatrixHistogramOption;
     errorMessage: string;
     headerChildren?: React.ReactNode;
+    footerChildren?: React.ReactNode;
     hideHistogramIfEmpty?: boolean;
     histogramType: MatrixHistogramType;
     id: string;
@@ -47,6 +48,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps &
     subtitle?: string | GetSubTitle;
     timelineId?: string;
     title: string | GetTitle;
+    yTitle?: string | undefined;
   };
 
 const DEFAULT_PANEL_HEIGHT = 300;
@@ -68,6 +70,7 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
   errorMessage,
   filterQuery,
   headerChildren,
+  footerChildren,
   histogramType,
   hideHistogramIfEmpty = false,
   id,
@@ -86,6 +89,7 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
   title,
   titleSize,
   yTickFormatter,
+  yTitle,
 }) => {
   const dispatch = useDispatch();
   const handleBrushEnd = useCallback(
@@ -114,8 +118,18 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
         onBrushEnd: handleBrushEnd,
         yTickFormatter,
         showLegend,
+        yTitle,
       }),
-    [chartHeight, startDate, legendPosition, endDate, handleBrushEnd, yTickFormatter, showLegend]
+    [
+      chartHeight,
+      startDate,
+      legendPosition,
+      endDate,
+      handleBrushEnd,
+      yTickFormatter,
+      showLegend,
+      yTitle,
+    ]
   );
   const [isInitialLoading, setIsInitialLoading] = useState(true);
   const [selectedStackByOption, setSelectedStackByOption] = useState<MatrixHistogramOption>(
@@ -229,6 +243,11 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
               timelineId={timelineId}
             />
           )}
+          {footerChildren != null && (
+            <EuiFlexGroup gutterSize="none" direction="row">
+              {footerChildren}
+            </EuiFlexGroup>
+          )}
         </HistogramPanel>
       </InspectButtonContainer>
       {showSpacer && <EuiSpacer data-test-subj="spacer" size="l" />}

x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts

@@ -70,6 +70,7 @@ export interface MatrixHistogramQueryProps {
   stackByField: string;
   startDate: string;
   histogramType: MatrixHistogramType;
+  threshold?: { field: string | undefined; value: number } | undefined;
 }
 
 export interface MatrixHistogramProps extends MatrixHistogramBasicProps {
@@ -104,6 +105,7 @@ export interface BarchartConfigs {
     yTickFormatter: TickFormatter;
     tickSize: number;
   };
+  yAxisTitle: string | undefined;
   settings: {
     legendPosition: Position;
     onBrushEnd: UpdateDateRange;

x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts

@@ -19,6 +19,7 @@ interface GetBarchartConfigsProps {
   onBrushEnd: UpdateDateRange;
   yTickFormatter?: (value: number) => string;
   showLegend?: boolean;
+  yTitle?: string | undefined;
 }
 
 export const DEFAULT_CHART_HEIGHT = 174;
@@ -32,6 +33,7 @@ export const getBarchartConfigs = ({
   onBrushEnd,
   yTickFormatter,
   showLegend,
+  yTitle,
 }: GetBarchartConfigsProps): BarchartConfigs => ({
   series: {
     xScaleType: ScaleType.Time,
@@ -43,6 +45,7 @@ export const getBarchartConfigs = ({
     yTickFormatter: yTickFormatter != null ? yTickFormatter : DEFAULT_Y_TICK_FORMATTER,
     tickSize: 8,
   },
+  yAxisTitle: yTitle,
   settings: {
     legendPosition: legendPosition ?? Position.Right,
     onBrushEnd,

x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts

@@ -5,7 +5,7 @@
  */
 
 import deepEqual from 'fast-deep-equal';
-import { noop } from 'lodash/fp';
+import { getOr, noop } from 'lodash/fp';
 import { useCallback, useEffect, useRef, useState } from 'react';
 
 import { MatrixHistogramQueryProps } from '../../components/matrix_histogram/types';
@@ -32,6 +32,10 @@ export interface UseMatrixHistogramArgs {
   inspect: InspectResponse;
   refetch: inputsModel.Refetch;
   totalCount: number;
+  buckets: Array<{
+    key: string;
+    doc_count: number;
+  }>;
 }
 
 const ID = 'matrixHistogramQuery';
@@ -44,6 +48,7 @@ export const useMatrixHistogram = ({
   indexNames,
   stackByField,
   startDate,
+  threshold,
 }: MatrixHistogramQueryProps): [boolean, UseMatrixHistogramArgs] => {
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
@@ -63,6 +68,7 @@ export const useMatrixHistogram = ({
       to: endDate,
     },
     stackByField,
+    threshold,
   });
 
   const [matrixHistogramResponse, setMatrixHistogramResponse] = useState<UseMatrixHistogramArgs>({
@@ -73,6 +79,7 @@ export const useMatrixHistogram = ({
     },
     refetch: refetch.current,
     totalCount: -1,
+    buckets: [],
   });
 
   const hostsSearch = useCallback(
@@ -91,13 +98,18 @@ export const useMatrixHistogram = ({
             next: (response) => {
               if (isCompleteResponse(response)) {
                 if (!didCancel) {
+                  const histogramBuckets: Array<{
+                    key: string;
+                    doc_count: number;
+                  }> = getOr([], 'rawResponse.aggregations.eventActionGroup.buckets', response);
                   setLoading(false);
                   setMatrixHistogramResponse((prevResponse) => ({
                     ...prevResponse,
                     data: response.matrixHistogramData,
                     inspect: getInspectResponse(response, prevResponse.inspect),
                     refetch: refetch.current,
                     totalCount: response.totalCount,
+                    buckets: histogramBuckets,
                   }));
                 }
                 searchSubscription$.unsubscribe();
@@ -144,13 +156,14 @@ export const useMatrixHistogram = ({
           to: endDate,
         },
         stackByField,
+        threshold,
       };
       if (!deepEqual(prevRequest, myRequest)) {
         return myRequest;
       }
       return prevRequest;
     });
-  }, [indexNames, endDate, filterQuery, startDate, stackByField, histogramType]);
+  }, [indexNames, endDate, filterQuery, startDate, stackByField, histogramType, threshold]);
 
   useEffect(() => {
     hostsSearch(matrixHistogramRequest);

x-pack/plugins/security_solution/public/common/hooks/eql/api.ts

@@ -3,11 +3,21 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { Unit } from '@elastic/datemath';
 
 import { HttpStart } from '../../../../../../../src/core/public';
 import { DETECTION_ENGINE_EQL_VALIDATION_URL } from '../../../../common/constants';
 import { EqlValidationSchema as EqlValidationRequest } from '../../../../common/detection_engine/schemas/request/eql_validation_schema';
 import { EqlValidationSchema as EqlValidationResponse } from '../../../../common/detection_engine/schemas/response/eql_validation_schema';
+import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public';
+import {
+  EqlSearchStrategyRequest,
+  EqlSearchStrategyResponse,
+} from '../../../../../data_enhanced/common';
+import { getEqlAggsData, getSequenceAggs } from './helpers';
+import { EqlPreviewResponse, Source } from './types';
+import { hasEqlSequenceQuery } from '../../../../common/detection_engine/utils';
+import { EqlSearchResponse } from '../../../../common/detection_engine/types';
 
 interface ApiParams {
   http: HttpStart;
@@ -29,3 +39,64 @@ export const validateEql = async ({
     signal,
   });
 };
+
+interface AggsParams extends EqlValidationRequest {
+  data: DataPublicPluginStart;
+  interval: Unit;
+  fromTime: string;
+  toTime: string;
+  signal: AbortSignal;
+}
+
+export const getEqlPreview = async ({
+  data,
+  index,
+  interval,
+  query,
+  fromTime,
+  toTime,
+  signal,
+}: AggsParams): Promise<EqlPreviewResponse> => {
+  try {
+    const response = await data.search
+      .search<EqlSearchStrategyRequest, EqlSearchStrategyResponse<EqlSearchResponse<Source>>>(
+        {
+          params: {
+            // @ts-expect-error allow_no_indices is missing on EqlSearch
+            allow_no_indices: true,
+            index: index.join(),
+            body: {
+              filter: {
+                range: {
+                  '@timestamp': {
+                    gte: toTime,
+                    lte: fromTime,
+                    format: 'strict_date_optional_time',
+                  },
+                },
+              },
+              query,
+              // EQL requires a cap, otherwise it defaults to 10
+              // It also sorts on ascending order, capping it at
+              // something smaller like 20, made it so that some of
+              // the more recent events weren't returned
+              size: 100,
+            },
+          },
+        },
+        {
+          strategy: 'eql',
+          abortSignal: signal,
+        }
+      )
+      .toPromise();
+
+    if (hasEqlSequenceQuery(query)) {
+      return getSequenceAggs(response, interval, toTime, fromTime);
+    } else {
+      return getEqlAggsData(response, interval, toTime, fromTime);
+    }
+  } catch (err) {
+    throw new Error(JSON.stringify(err));
+  }
+};