[ML] DF Analytics Regression exploration: replace table with data grid (#63650) (#63739)

* add feature_importance column correctly

* wip: switch regression table to datagrid

* add search bar to regression view

* ensure feature importance fields show up correctly

* wip: filter by training/testing

* remove separate testing/training filter

* make error more clear

* handle lucene queries

* remove unnecessary comment

* ensure boolean shows up correctly.no sorting by feature importance

* remove unused translations

Author: alvarezmelissa87

x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts

@@ -13,7 +13,6 @@ import { ml } from '../../services/ml_api_service';
 import { Dictionary } from '../../../../common/types/common';
 import { getErrorMessage } from '../../../../common/util/errors';
 import { SavedSearchQuery } from '../../contexts/ml';
-import { SortDirection } from '../../components/ml_in_memory_table';
 
 export type IndexName = string;
 export type IndexPattern = string;
@@ -53,13 +52,9 @@ export interface ClassificationAnalysis {
   classification: Classification;
 }
 
-export interface LoadExploreDataArg {
-  field: string;
-  direction: SortDirection;
+export interface LoadRegressionExploreDataArg {
+  filterByIsTraining?: boolean;
   searchQuery: SavedSearchQuery;
-  requiresKeyword?: boolean;
-  pageIndex?: number;
-  pageSize?: number;
 }
 
 export const SEARCH_SIZE = 1000;
@@ -272,6 +267,11 @@ export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuer
   return keys.length === 1 && keys[0] === 'bool';
 };
 
+export const isQueryStringQuery = (arg: any): arg is QueryStringQuery => {
+  const keys = Object.keys(arg);
+  return keys.length === 1 && keys[0] === 'query_string';
+};
+
 export const isRegressionEvaluateResponse = (arg: any): arg is RegressionEvaluateResponse => {
   const keys = Object.keys(arg);
   return (
@@ -396,6 +396,10 @@ interface ResultsSearchTermQuery {
   term: Dictionary<any>;
 }
 
+interface QueryStringQuery {
+  query_string: Dictionary<any>;
+}
+
 export type ResultsSearchQuery = ResultsSearchBoolQuery | ResultsSearchTermQuery | SavedSearchQuery;
 
 export function getEvalQueryBody({
@@ -409,16 +413,34 @@ export function getEvalQueryBody({
   searchQuery?: ResultsSearchQuery;
   ignoreDefaultQuery?: boolean;
 }) {
-  let query: ResultsSearchQuery = {
+  let query;
+
+  const trainingQuery: ResultsSearchQuery = {
     term: { [`${resultsField}.is_training`]: { value: isTraining } },
   };
 
-  if (searchQuery !== undefined && ignoreDefaultQuery === true) {
-    query = searchQuery;
-  } else if (searchQuery !== undefined && isResultsSearchBoolQuery(searchQuery)) {
-    const searchQueryClone = cloneDeep(searchQuery);
-    searchQueryClone.bool.must.push(query);
+  const searchQueryClone = cloneDeep(searchQuery);
+
+  if (isResultsSearchBoolQuery(searchQueryClone)) {
+    if (searchQueryClone.bool.must === undefined) {
+      searchQueryClone.bool.must = [];
+    }
+
+    searchQueryClone.bool.must.push(trainingQuery);
     query = searchQueryClone;
+  } else if (isQueryStringQuery(searchQueryClone)) {
+    query = {
+      bool: {
+        must: [searchQueryClone, trainingQuery],
+      },
+    };
+  } else {
+    // Not a bool or string query so we need to create it so can add the trainingQuery
+    query = {
+      bool: {
+        must: [trainingQuery],
+      },
+    };
   }
   return query;
 }

x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts

@@ -246,8 +246,15 @@ export const getDefaultFieldsFromJobCaps = (
   fields: Field[],
   jobConfig: DataFrameAnalyticsConfig,
   needsDestIndexFields: boolean
-): { selectedFields: Field[]; docFields: Field[]; depVarType?: ES_FIELD_TYPES } => {
-  const fieldsObj = { selectedFields: [], docFields: [] };
+): {
+  selectedFields: Field[];
+  docFields: Field[];
+  depVarType?: ES_FIELD_TYPES;
+} => {
+  const fieldsObj = {
+    selectedFields: [],
+    docFields: [],
+  };
   if (fields.length === 0) {
     return fieldsObj;
   }
@@ -267,38 +274,37 @@ export const getDefaultFieldsFromJobCaps = (
   const featureImportanceFields = [];
 
   if ((numTopFeatureImportanceValues ?? 0) > 0) {
-    featureImportanceFields.push(
-      ...fields.map(d => ({
-        id: `${resultsField}.feature_importance.${d.id}`,
-        name: `${resultsField}.feature_importance.${d.name}`,
-        type: KBN_FIELD_TYPES.NUMBER,
-      }))
-    );
+    featureImportanceFields.push({
+      id: `${resultsField}.feature_importance`,
+      name: `${resultsField}.feature_importance`,
+      type: KBN_FIELD_TYPES.NUMBER,
+    });
   }
 
+  let allFields: any = [];
   // Only need to add these fields if we didn't use dest index pattern to get the fields
-  const allFields: any =
-    needsDestIndexFields === true
-      ? [
-          {
-            id: `${resultsField}.is_training`,
-            name: `${resultsField}.is_training`,
-            type: ES_FIELD_TYPES.BOOLEAN,
-          },
-          { id: predictedField, name: predictedField, type },
-          ...featureImportanceFields,
-        ]
-      : [];
-
-  allFields.push(...fields);
+  if (needsDestIndexFields === true) {
+    allFields.push(
+      {
+        id: `${resultsField}.is_training`,
+        name: `${resultsField}.is_training`,
+        type: ES_FIELD_TYPES.BOOLEAN,
+      },
+      { id: predictedField, name: predictedField, type }
+    );
+  }
+
+  allFields.push(...fields, ...featureImportanceFields);
   allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) =>
     sortRegressionResultsFields(a, b, jobConfig)
   );
+  // Remove feature_importance fields provided by dest index since feature_importance is an array the path is not valid
+  if (needsDestIndexFields === false) {
+    allFields = allFields.filter((field: any) => !field.name.includes('.feature_importance.'));
+  }
 
   let selectedFields = allFields.filter(
-    (field: any) =>
-      field.name === predictedField ||
-      (!field.name.includes('.keyword') && !field.name.includes('.feature_importance.'))
+    (field: any) => field.name === predictedField || !field.name.includes('.keyword')
   );
 
   if (selectedFields.length > DEFAULT_REGRESSION_COLUMNS) {

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts

@@ -22,7 +22,6 @@ import { newJobCapsService } from '../../../../../services/new_job_capabilities_
 import { Field } from '../../../../../../../common/types/fields';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
 import {
-  LoadExploreDataArg,
   defaultSearchQuery,
   ResultsSearchQuery,
   isResultsSearchBoolQuery,
@@ -37,12 +36,23 @@ import {
   SEARCH_SIZE,
   SearchQuery,
 } from '../../../../common';
+import { SavedSearchQuery } from '../../../../../contexts/ml';
+
+interface LoadClassificationExploreDataArg {
+  direction: SortDirection;
+  filterByIsTraining?: boolean;
+  field: string;
+  searchQuery: SavedSearchQuery;
+  requiresKeyword?: boolean;
+  pageIndex?: number;
+  pageSize?: number;
+}
 
 export type TableItem = Record<string, any>;
 
 export interface UseExploreDataReturnType {
   errorMessage: string;
-  loadExploreData: (arg: LoadExploreDataArg) => void;
+  loadExploreData: (arg: LoadClassificationExploreDataArg) => void;
   sortField: EsFieldName;
   sortDirection: SortDirection;
   status: INDEX_STATUS;
@@ -84,7 +94,7 @@ export const useExploreData = (
     direction,
     searchQuery,
     requiresKeyword,
-  }: LoadExploreDataArg) => {
+  }: LoadClassificationExploreDataArg) => {
     if (jobConfig !== undefined) {
       setErrorMessage('');
       setStatus(INDEX_STATUS.LOADING);