Implement new Kibana query language (#12624)

Initial version of an experimental new query language for Kibana.

Author: Bargs

docs/management/advanced-options.asciidoc

@@ -22,6 +22,8 @@ compatible with other configuration settings. Deleting a custom setting removes
 .Kibana Settings Reference
 [horizontal]
 `query:queryString:options`:: Options for the Lucene query string parser.
+`search:queryLanguage`:: Default is `lucene`. Query language used by the query bar. Choose between the lucene query syntax and kuery, an experimental new language built specifically for Kibana.
+`search:queryLanguage:switcher:enable`:: Show or hide the query language switcher in the query bar.
 `sort:options`:: Options for the Elasticsearch {ref}/search-request-sort.html[sort] parameter.
 `dateFormat`:: The format to use for displaying pretty-formatted dates.
 `dateFormat:tz`:: The timezone that Kibana uses. The default value of `Browser` uses the timezone detected by the browser.

src/core_plugins/kibana/public/context/api/__tests__/anchor.js

@@ -88,9 +88,12 @@ describe('context app', function () {
           const setQuerySpy = searchSourceStub.set.withArgs('query');
           expect(setQuerySpy.calledOnce).to.be(true);
           expect(setQuerySpy.firstCall.args[1]).to.eql({
-            terms: {
-              _uid: ['UID'],
+            query: {
+              terms: {
+                _uid: ['UID'],
+              }
             },
+            language: 'lucene'
           });
         });
     });

src/core_plugins/kibana/public/context/api/anchor.js

@@ -15,9 +15,12 @@ function fetchAnchorProvider(courier, Private) {
       .set('version', true)
       .set('size', 1)
       .set('query', {
-        terms: {
-          _uid: [uid],
+        query: {
+          terms: {
+            _uid: [uid],
+          }
         },
+        language: 'lucene'
       })
       .set('sort', sort);
 

src/core_plugins/kibana/public/context/api/context.js

@@ -50,7 +50,10 @@ function fetchContextProvider(courier, Private) {
       .set('size', size)
       .set('filter', filters)
       .set('query', {
-        match_all: {},
+        query: {
+          match_all: {},
+        },
+        language: 'lucene'
       })
       .set('searchAfter', anchorDocument.sort)
       .set('sort', sort);

src/core_plugins/kibana/public/dashboard/dashboard.html

@@ -31,54 +31,14 @@
       </div>
 
       <!-- Search. -->
-      <form
-        data-transclude-slot="bottomRow"
-        class="fullWidth"
-        ng-show="chrome.getVisible()"
-        role="form"
-        name="queryInput"
-        ng-submit="filterResults()"
-      >
-        <div class="typeahead" kbn-typeahead="dashboard" on-select="filterResults()">
-          <div class="kuiLocalSearch">
-            <div class="kuiLocalSearchAssistedInput">
-              <input
-                parse-query
-                input-focus
-                kbn-typeahead-input
-                ng-model="model.query"
-                placeholder="Search... (e.g. status:200 AND extension:PHP)"
-                aria-label="Enter query"
-                data-test-subj="dashboardQuery"
-                type="text"
-                class="kuiLocalSearchInput kuiLocalSearchInput--lucene"
-                ng-class="{'kuiLocalSearchInput-isInvalid': queryInput.$invalid}"
-              >
-              <div class="kuiLocalSearchAssistedInput__assistance">
-                <p class="kuiText">
-                  <a
-                    class="kuiLink"
-                    ng-href="{{queryDocLinks.luceneQuerySyntax}}"
-                    target="_blank"
-                  >
-                    Uses lucene query syntax
-                  </a>
-                </p>
-              </div>
-            </div>
-            <button
-              type="submit"
-              aria-label="Submit query"
-              class="kuiLocalSearchButton"
-              data-test-subj="dashboardQueryFilterButton"
-              ng-disabled="queryInput.$invalid"
-            >
-              <span class="kuiIcon fa-search" aria-hidden="true"></span>
-            </button>
-          </div>
-          <kbn-typeahead-items></kbn-typeahead-items>
-        </div>
-      </form>
+      <div ng-show="chrome.getVisible()" class="fullWidth" data-transclude-slot="bottomRow">
+        <query-bar
+          query="model.query"
+          app-name="'dashboard'"
+          on-submit="updateQuery($query)"
+        >
+        </query-bar>
+      </div>
     </div>
   </kbn-top-nav>
 
@@ -87,6 +47,7 @@
     ng-show="showFilterBar()"
     state="state"
     index-patterns="indexPatterns"
+    ng-if="model.query.language === 'lucene'"
   ></filter-bar>
 
   <div
@@ -126,6 +87,7 @@ <h2 class="kuiTitle kuiVerticalRhythm">
     toggle-expand="toggleExpandPanel"
     register-panel-index-pattern="registerPanelIndexPattern"
     data-shared-items-count="{{panels.length}}"
+    on-filter="filter"
   ></dashboard-grid>
 
   <dashboard-panel

src/core_plugins/kibana/public/dashboard/dashboard.js

@@ -6,6 +6,7 @@ import chrome from 'ui/chrome';
 
 import 'plugins/kibana/dashboard/grid';
 import 'plugins/kibana/dashboard/panel/panel';
+import 'ui/query_bar';
 
 import { SavedObjectNotFound } from 'ui/errors';
 import { getDashboardTitle, getUnsavedChangesWarningMessage } from './dashboard_strings';
@@ -25,6 +26,8 @@ import { notify } from 'ui/notify';
 import './panel/get_object_loaders_for_dashboard';
 import { documentationLinks } from 'ui/documentation_links/documentation_links';
 import { showCloneModal } from './top_nav/show_clone_modal';
+import { migrateLegacyQuery } from 'ui/utils/migrateLegacyQuery';
+import { QueryManagerProvider } from 'ui/query_manager';
 import { ESC_KEY_CODE } from 'ui_framework/services';
 
 const app = uiModules.get('app/dashboard', [
@@ -81,6 +84,7 @@ app.directive('dashboardApp', function ($injector) {
   const quickRanges = $injector.get('quickRanges');
   const kbnUrl = $injector.get('kbnUrl');
   const confirmModal = $injector.get('confirmModal');
+  const config = $injector.get('config');
   const Private = $injector.get('Private');
   const brushEvent = Private(UtilsBrushEventProvider);
   const filterBarClickHandler = Private(FilterBarClickHandlerProvider);
@@ -100,6 +104,7 @@ app.directive('dashboardApp', function ($injector) {
       }
 
       const dashboardState = new DashboardState(dash, AppState, dashboardConfig);
+      const queryManager = Private(QueryManagerProvider)(dashboardState.getAppState());
 
       // The 'previouslyStored' check is so we only update the time filter on dashboard open, not during
       // normal cross app navigation.
@@ -130,7 +135,10 @@ app.directive('dashboardApp', function ($injector) {
         updateState();
       });
 
-      dashboardState.applyFilters(dashboardState.getQuery(), filterBar.getFilters());
+      dashboardState.applyFilters(
+        dashboardState.getQuery() || { query: '', language: config.get('search:queryLanguage') },
+        filterBar.getFilters()
+      );
       let pendingVisCount = _.size(dashboardState.getPanels());
 
       timefilter.enabled = true;
@@ -179,9 +187,14 @@ app.directive('dashboardApp', function ($injector) {
         }
       };
 
-      $scope.filterResults = function () {
-        dashboardState.applyFilters($scope.model.query, filterBar.getFilters());
-        $scope.refresh();
+      $scope.updateQuery = function (query) {
+        // reset state if language changes
+        if ($scope.model.query.language && $scope.model.query.language !== query.language) {
+          filterBar.removeAll();
+          dashboardState.getAppState().$newFilters = [];
+        }
+
+        $scope.model.query = query;
       };
 
       // called by the saved-object-finder when a user clicks a vis
@@ -229,6 +242,28 @@ app.directive('dashboardApp', function ($injector) {
         $scope.indexPatterns = dashboardState.getPanelIndexPatterns();
       };
 
+      $scope.filter = function (field, value, operator, index) {
+        queryManager.add(field, value, operator, index);
+        updateState();
+      };
+
+
+      $scope.$watch('model.query', (newQuery) => {
+        $scope.model.query = migrateLegacyQuery(newQuery);
+        dashboardState.applyFilters($scope.model.query, filterBar.getFilters());
+        $scope.refresh();
+      });
+
+      $scope.$watchCollection(() => dashboardState.getAppState().$newFilters, function (filters = []) {
+        // need to convert filters generated from user interaction with viz into kuery AST
+        // These are handled by the filter bar directive when lucene is the query language
+        Promise.all(filters.map(queryManager.addLegacyFilter))
+          .then(() => dashboardState.getAppState().$newFilters = [])
+          .then(updateState)
+          .then(() => dashboardState.applyFilters($scope.model.query, filterBar.getFilters()))
+          .then($scope.refresh());
+      });
+
       $scope.$listen(timefilter, 'fetch', $scope.refresh);
 
       function updateViewMode(newMode) {

src/core_plugins/kibana/public/dashboard/dashboard_context.js

@@ -2,16 +2,19 @@
 import _ from 'lodash';
 import { FilterBarQueryFilterProvider } from 'ui/filter_bar/query_filter';
 import 'ui/state_management/app_state';
+import { luceneStringToDsl } from '../../../../ui/public/courier/data_source/build_query/lucene_string_to_dsl';
 
 export function dashboardContextProvider(Private, getAppState) {
   return () => {
     const queryFilter = Private(FilterBarQueryFilterProvider);
     const bool = { must: [], must_not: [] };
     const filterBarFilters = queryFilter.getFilters();
-    const queryBarFilter = getAppState().query;
+    const queryBarQuery = getAppState().query;
 
-    // Add the query bar filter, its handled differently.
-    bool.must.push(queryBarFilter);
+    if (queryBarQuery.language === 'lucene') {
+      // Add the query bar filter, its handled differently.
+      bool.must.push(luceneStringToDsl(queryBarQuery.query));
+    }
 
     // Add each of the filter bar filters
     _.each(filterBarFilters, function (filter) {

src/core_plugins/kibana/public/dashboard/dashboard_state.js

@@ -219,7 +219,19 @@ export class DashboardState {
    * new dashboard, if the query differs from the default.
    */
   getQueryChanged() {
-    return !_.isEqual(this.appState.query, this.getLastSavedQuery());
+    const currentQuery = this.appState.query;
+    const lastSavedQuery = this.getLastSavedQuery();
+
+    const isLegacyStringQuery = (
+      _.isString(lastSavedQuery)
+      && _.isPlainObject(currentQuery)
+      && _.has(currentQuery, 'query')
+    );
+    if (isLegacyStringQuery) {
+      return lastSavedQuery !== currentQuery.query;
+    }
+
+    return !_.isEqual(currentQuery, lastSavedQuery);
   }
 
   /**
@@ -404,14 +416,8 @@ export class DashboardState {
    */
   applyFilters(query, filters) {
     this.appState.query = query;
-    if (this.appState.query) {
-      this.savedDashboard.searchSource.set('filter', _.union(filters, [{
-        query: this.appState.query
-      }]));
-    } else {
-      this.savedDashboard.searchSource.set('filter', filters);
-    }
-
+    this.savedDashboard.searchSource.set('query', query);
+    this.savedDashboard.searchSource.set('filter', filters);
     this.saveState();
   }
 
@@ -424,6 +430,8 @@ export class DashboardState {
     this.stateMonitor.ignoreProps('viewMode');
     // Filters need to be compared manually because they sometimes have a $$hashkey stored on the object.
     this.stateMonitor.ignoreProps('filters');
+    // Query needs to be compared manually because saved legacy queries get migrated in app state automatically
+    this.stateMonitor.ignoreProps('query');
 
     this.stateMonitor.onChange(status => {
       this.isDirty = status.dirty;

src/core_plugins/kibana/public/dashboard/filter_utils.js

@@ -33,10 +33,13 @@ export class FilterUtils {
    * @returns {QueryFilter}
    */
   static getQueryFilterForDashboard(dashboard) {
-    const defaultQueryFilter = { query_string: { query: '*' } };
+    if (dashboard.searchSource.getOwn('query')) {
+      return dashboard.searchSource.getOwn('query');
+    }
+
     const dashboardFilters = this.getDashboardFilters(dashboard);
     const dashboardQueryFilter = _.find(dashboardFilters, this.isQueryFilter);
-    return dashboardQueryFilter ? dashboardQueryFilter.query : defaultQueryFilter;
+    return dashboardQueryFilter ? dashboardQueryFilter.query : '';
   }
 
   /**

src/core_plugins/kibana/public/dashboard/grid.js

@@ -60,6 +60,11 @@ app.directive('dashboardGrid', function ($compile, Notifier) {
        * @type {function}
        */
       toggleExpand: '=',
+      /**
+       * Called when a filter action has been triggered by a panel
+       * @type {function}
+       */
+      onFilter: '=',
     },
     link: function ($scope, $el) {
       const notify = new Notifier();
@@ -223,7 +228,8 @@ app.directive('dashboardGrid', function ($compile, Notifier) {
                   app-state="appState"
                   register-panel-index-pattern="registerPanelIndexPattern"
                   toggle-expand="toggleExpand(${panel.panelIndex})"
-                  create-child-ui-state="createChildUiState">
+                  create-child-ui-state="createChildUiState"
+                  on-filter="onFilter">
             </li>`;
         const panelElement = $compile(panelHtml)($scope);
         panelElementMapping[panel.panelIndex] = panelElement;