[SIEM] Phase I - Add saved query in SIEM solution (#47306)

* Add Search Bar components
Integration of the Search Bar component in host and network page
Fix state URL with new Search Bar

* update unit test

* Fix URL state to match Discover + Fix ML to match with new url state + fix cypress test

* fix behavior when save as new query

* savedQuery - do not try to update date picker when there is no timefilter

* fix refresh

* some merge issue + fix back to active page to zero

* review I

* hack to remove lag

* fix type

Author: XavierM

src/legacy/core_plugins/data/public/query/query_bar/components/query_bar_top_row.tsx

@@ -24,7 +24,7 @@ import React, { useState, useEffect } from 'react';
 
 import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiLink, EuiSuperDatePicker } from '@elastic/eui';
 // @ts-ignore
-import { EuiSuperUpdateButton } from '@elastic/eui';
+import { EuiSuperUpdateButton, OnRefreshProps } from '@elastic/eui';
 import { FormattedMessage, InjectedIntl, injectI18n } from '@kbn/i18n/react';
 import { Toast } from 'src/core/public';
 import { TimeRange } from 'src/plugins/data/public';
@@ -41,10 +41,12 @@ interface Props {
   query?: Query;
   onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void;
   onChange: (payload: { dateRange: TimeRange; query?: Query }) => void;
+  onRefresh?: (payload: { dateRange: TimeRange }) => void;
   disableAutoFocus?: boolean;
   screenTitle?: string;
   indexPatterns?: Array<IndexPattern | string>;
   intl: InjectedIntl;
+  isLoading?: boolean;
   prepend?: React.ReactNode;
   showQueryInput?: boolean;
   showDatePicker?: boolean;
@@ -125,6 +127,18 @@ function QueryBarTopRowUI(props: Props) {
     }
   }
 
+  function onRefresh({ start, end }: OnRefreshProps) {
+    const retVal = {
+      dateRange: {
+        from: start,
+        to: end,
+      },
+    };
+    if (props.onRefresh) {
+      props.onRefresh(retVal);
+    }
+  }
+
   function onSubmit({ query, dateRange }: { query?: Query; dateRange: TimeRange }) {
     handleLuceneSyntaxWarning();
 
@@ -175,6 +189,7 @@ function QueryBarTopRowUI(props: Props) {
       <EuiSuperUpdateButton
         needsUpdate={props.isDirty}
         isDisabled={isDateRangeInvalid}
+        isLoading={props.isLoading}
         onClick={onClickSubmitButton}
         data-test-subj="querySubmitButton"
       />
@@ -227,6 +242,7 @@ function QueryBarTopRowUI(props: Props) {
           isPaused={props.isRefreshPaused}
           refreshInterval={props.refreshInterval}
           onTimeChange={onTimeChange}
+          onRefresh={onRefresh}
           onRefreshChange={props.onRefreshChange}
           showUpdateButton={false}
           recentlyUsedRanges={recentlyUsedRanges}

src/legacy/core_plugins/data/public/search/search_bar/components/create_search_bar.tsx

@@ -17,7 +17,8 @@
  * under the License.
  */
 
-import React from 'react';
+import React, { useState, useEffect } from 'react';
+import { Subscription } from 'rxjs';
 import { Filter } from '@kbn/es-query';
 import { CoreStart } from 'src/core/public';
 import { DataPublicPluginStart } from 'src/plugins/data/public';
@@ -64,8 +65,48 @@ export function createSearchBar({
   // App name should come from the core application service.
   // Until it's available, we'll ask the user to provide it for the pre-wired component.
   return (props: StatetfulSearchBarProps) => {
+    const tfRefreshInterval = timefilter.timefilter.getRefreshInterval();
+    const fmFilters = filterManager.getFilters();
+    const [refreshInterval, setRefreshInterval] = useState(tfRefreshInterval.value);
+    const [refreshPaused, setRefreshPaused] = useState(tfRefreshInterval.pause);
+
+    const [filters, setFilters] = useState(fmFilters);
+
+    // We do not really need to keep track of the time
+    // since this is just for initialization
     const timeRange = timefilter.timefilter.getTime();
-    const refreshInterval = timefilter.timefilter.getRefreshInterval();
+
+    useEffect(() => {
+      let isSubscribed = true;
+      const subscriptions = new Subscription();
+      subscriptions.add(
+        timefilter.timefilter.getRefreshIntervalUpdate$().subscribe({
+          next: () => {
+            if (isSubscribed) {
+              const newRefreshInterval = timefilter.timefilter.getRefreshInterval();
+              setRefreshInterval(newRefreshInterval.value);
+              setRefreshPaused(newRefreshInterval.pause);
+            }
+          },
+        })
+      );
+
+      subscriptions.add(
+        filterManager.getUpdates$().subscribe({
+          next: () => {
+            if (isSubscribed) {
+              const newFilters = filterManager.getFilters();
+              setFilters(newFilters);
+            }
+          },
+        })
+      );
+
+      return () => {
+        isSubscribed = false;
+        subscriptions.unsubscribe();
+      };
+    }, []);
 
     return (
       <KibanaContextProvider
@@ -80,9 +121,9 @@ export function createSearchBar({
           timeHistory={timefilter.history}
           dateRangeFrom={timeRange.from}
           dateRangeTo={timeRange.to}
-          refreshInterval={refreshInterval.value}
-          isRefreshPaused={refreshInterval.pause}
-          filters={filterManager.getFilters()}
+          refreshInterval={refreshInterval}
+          isRefreshPaused={refreshPaused}
+          filters={filters}
           onFiltersUpdated={defaultFiltersUpdated(filterManager)}
           onRefreshChange={defaultOnRefreshChange(timefilter)}
           {...props}

src/legacy/core_plugins/data/public/search/search_bar/components/search_bar.tsx

@@ -58,6 +58,7 @@ interface SearchBarInjectedDeps {
 
 export interface SearchBarOwnProps {
   indexPatterns?: IndexPattern[];
+  isLoading?: boolean;
   customSubmitButton?: React.ReactNode;
   screenTitle?: string;
 
@@ -79,6 +80,8 @@ export interface SearchBarOwnProps {
   onSavedQueryUpdated?: (savedQuery: SavedQuery) => void;
   // User has cleared the active query, your app should clear the entire query bar
   onClearSavedQuery?: () => void;
+
+  onRefresh?: (payload: { dateRange: TimeRange }) => void;
 }
 
 export type SearchBarProps = SearchBarOwnProps & SearchBarInjectedDeps;
@@ -377,6 +380,7 @@ class SearchBarUI extends Component<SearchBarProps, State> {
           screenTitle={this.props.screenTitle}
           onSubmit={this.onQueryBarSubmit}
           indexPatterns={this.props.indexPatterns}
+          isLoading={this.props.isLoading}
           prepend={this.props.showFilterBar ? savedQueryManagement : undefined}
           showDatePicker={this.props.showDatePicker}
           dateRangeFrom={this.state.dateRangeFrom}
@@ -385,6 +389,7 @@ class SearchBarUI extends Component<SearchBarProps, State> {
           refreshInterval={this.props.refreshInterval}
           showAutoRefreshOnly={this.props.showAutoRefreshOnly}
           showQueryInput={this.props.showQueryInput}
+          onRefresh={this.props.onRefresh}
           onRefreshChange={this.props.onRefreshChange}
           onChange={this.onQueryBarChange}
           isDirty={this.isDirty()}

src/legacy/core_plugins/data/public/search/search_bar/index.tsx

@@ -23,7 +23,7 @@ import { Query } from '../../query/query_bar';
 
 export * from './components';
 
-type SavedQueryTimeFilter = TimeRange & {
+export type SavedQueryTimeFilter = TimeRange & {
   refreshInterval: RefreshInterval;
 };
 

x-pack/legacy/plugins/siem/cypress/integration/lib/ml_conditional_links/index.ts

@@ -7,10 +7,10 @@
 /*
  * These links are for different test scenarios that try and capture different drill downs into
  * ml-network and ml-hosts and are of the flavor of testing:
- * A filter being null: (filterQuery:!n)
- * A filter being set with single values: filterQuery:(expression:%27process.name%20:%20%22conhost.exe%22%27,kind:kuery)
- * A filter being set with multiple values: filterQuery:(expression:%27process.name%20:%20%22conhost.exe,sc.exe%22%27,kind:kuery)
- * A filter containing variables not replaced: filterQuery:(expression:%27process.name%20:%20%$process.name$%22%27,kind:kuery)
+ * A filter being null: (query:!n)
+ * A filter being set with single values: query=(query:%27process.name%20:%20%22conhost.exe%22%27,language:kuery)
+ * A filter being set with multiple values:  query=(query:%27process.name%20:%20%22conhost.exe,sc.exe%22%27,language:kuery)
+ * A filter containing variables not replaced:  query=(query:%27process.name%20:%20%$process.name$%22%27,language:kuery)
  *
  * In different combination with:
  * network not being set: $ip$
@@ -23,54 +23,54 @@
  * host having multiple values: suricata-iowa,siem-windows
  */
 
-// Single IP with a null for the filterQuery:
-export const mlNetworkSingleIpNullFilterQuery =
-  "/app/siem#/ml-network/ip/127.0.0.1?kqlQuery=(filterQuery:!n,queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// Single IP with a null for the Query:
+export const mlNetworkSingleIpNullKqlQuery =
+  "/app/siem#/ml-network/ip/127.0.0.1?query=!n&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// Single IP with a value for the filterQuery:
-export const mlNetworkSingleIpFilterQuery =
-  "/app/siem#/ml-network/ip/127.0.0.1?kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// Single IP with a value for the Query:
+export const mlNetworkSingleIpKqlQuery =
+  "/app/siem#/ml-network/ip/127.0.0.1?query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// Multiple IPs with a null for the filterQuery:
-export const mlNetworkMultipleIpNullFilterQuery =
-  "/app/siem#/ml-network/ip/127.0.0.1,127.0.0.2?kqlQuery=(filterQuery:!n,queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// Multiple IPs with a null for the Query:
+export const mlNetworkMultipleIpNullKqlQuery =
+  "/app/siem#/ml-network/ip/127.0.0.1,127.0.0.2?query=!n&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// Multiple IPs with a value for the filterQuery:
-export const mlNetworkMultipleIpFilterQuery =
-  "/app/siem#/ml-network/ip/127.0.0.1,127.0.0.2?kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// Multiple IPs with a value for the Query:
+export const mlNetworkMultipleIpKqlQuery =
+  "/app/siem#/ml-network/ip/127.0.0.1,127.0.0.2?query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// $ip$ with a null filterQuery:
-export const mlNetworkNullFilterQuery =
-  "/app/siem#/ml-network/ip/$ip$?kqlQuery=(filterQuery:!n,queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// $ip$ with a null Query:
+export const mlNetworkNullKqlQuery =
+  "/app/siem#/ml-network/ip/$ip$?query=!n&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// $ip$ with a value for the filterQuery:
-export const mlNetworkFilterQuery =
-  "/app/siem#/ml-network/ip/$ip$?kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
+// $ip$ with a value for the Query:
+export const mlNetworkKqlQuery =
+  "/app/siem#/ml-network/ip/$ip$?query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-28T11:00:00.000Z',kind:absolute,to:'2019-08-28T13:59:59.999Z')))";
 
-// Single host name with a null for the filterQuery:
-export const mlHostSingleHostNullFilterQuery =
-  "/app/siem#/ml-hosts/siem-windows?_g=()&kqlQuery=(filterQuery:!n,queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Single host name with a null for the Query:
+export const mlHostSingleHostNullKqlQuery =
+  "/app/siem#/ml-hosts/siem-windows?_g=()&query=!n&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
-// Single host name with a variable in the filterQuery
-export const mlHostSingleHostFilterQueryVariable =
-  "/app/siem#/ml-hosts/siem-windows?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22$process.name$%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Single host name with a variable in the Query:
+export const mlHostSingleHostKqlQueryVariable =
+  "/app/siem#/ml-hosts/siem-windows?_g=()&query=(language:kuery,query:'process.name%20:%20%22$process.name$%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
-// Single host name with a value for filterQuery:
-export const mlHostSingleHostFilterQuery =
-  "/app/siem#/ml-hosts/siem-windows?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Single host name with a value for Query:
+export const mlHostSingleHostKqlQuery =
+  "/app/siem#/ml-hosts/siem-windows?_g=()&query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
-// Multiple host names with null for filterQuery
-export const mlHostMultiHostNullFilterQuery =
-  "/app/siem#/ml-hosts/siem-windows,siem-suricata?_g=()&kqlQuery=(filterQuery:!n,queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Multiple host names with null for Query:
+export const mlHostMultiHostNullKqlQuery =
+  "/app/siem#/ml-hosts/siem-windows,siem-suricata?_g=()&query=!n&&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
-// Multiple host names with a value for filterQuery
-export const mlHostMultiHostFilterQuery =
-  "/app/siem#/ml-hosts/siem-windows,siem-suricata?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Multiple host names with a value for Query:
+export const mlHostMultiHostKqlQuery =
+  "/app/siem#/ml-hosts/siem-windows,siem-suricata?_g=()&query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
 // Undefined/null host name with a null for the KQL:
-export const mlHostVariableHostNullFilterQuery =
-  "/app/siem#/ml-hosts/$host.name$?_g=()&kqlQuery=(filterQuery:!n,queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+export const mlHostVariableHostNullKqlQuery =
+  "/app/siem#/ml-hosts/$host.name$?_g=()&query=!n&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
 
-// Undefined/null host name but with a value for filterQuery
-export const mlHostVariableHostFilterQuery =
-  "/app/siem#/ml-hosts/$host.name$?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22conhost.exe,sc.exe%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";
+// Undefined/null host name but with a value for Query:
+export const mlHostVariableHostKqlQuery =
+  "/app/siem#/ml-hosts/$host.name$?_g=()&query=(language:kuery,query:'process.name%20:%20%22conhost.exe,sc.exe%22')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-06-06T06:00:00.000Z',kind:absolute,to:'2019-06-07T05:59:59.999Z')))";