[ML] Data Frames: Update stats data structure. (#42117) (#42147)

Fixes UI regressions caused by breaking API changes.

Author: walterra

x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/expanded_row.tsx

@@ -4,11 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get } from 'lodash';
-
 import React from 'react';
 
 import { EuiBadge, EuiText } from '@elastic/eui';
+import { idx } from '@kbn/elastic-idx';
 
 import { getSelectableFields, EsDoc } from '../../../../common';
 
@@ -19,7 +18,8 @@ interface ExpandedRowProps {
 export const ExpandedRow: React.SFC<ExpandedRowProps> = ({ item }) => {
   const keys = getSelectableFields([item]);
   const list = keys.map(k => {
-    const value = get(item._source, k, '');
+    // split the attribute key string and use reduce with an idx check to access nested attributes.
+    const value = k.split('.').reduce((obj, i) => idx(obj, _ => _[i]), item._source) || '';
     return (
       <span key={k}>
         <EuiBadge>{k}:</EuiBadge>

x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/use_source_index_data.ts

@@ -4,12 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get } from 'lodash';
-
 import React, { useEffect, useState } from 'react';
 
 import { SearchResponse } from 'elasticsearch';
 
+import { idx } from '@kbn/elastic-idx';
+
 import { IndexPattern } from 'ui/index_patterns';
 
 import { ml } from '../../../../../services/ml_api_service';
@@ -82,7 +82,7 @@ export const useSourceIndexData = (
           [key: string]: any;
         };
         flattenedFields.forEach(ff => {
-          item[ff] = get(doc._source, ff);
+          item[ff] = idx(doc._source, _ => _[ff]);
           if (item[ff] === undefined) {
             item[ff] = doc._source[`"${ff}"`];
           }

x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx

@@ -5,6 +5,7 @@
  */
 
 import React, { Fragment, SFC, useContext, useEffect, useState } from 'react';
+import { idx } from '@kbn/elastic-idx';
 import { i18n } from '@kbn/i18n';
 import { toastNotifications } from 'ui/notify';
 
@@ -191,7 +192,12 @@ export const StepCreateForm: SFC<Props> = React.memo(
           try {
             const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId);
             if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) {
-              const percent = Math.round(stats.transforms[0].state.progress.percent_complete);
+              const percent = Math.round(
+                idx(
+                  stats,
+                  _ => _.transforms[0].checkpointing.next.checkpoint_progress.percent_complete
+                ) || 0
+              );
               setProgressPercentComplete(percent);
               if (percent >= 100) {
                 clearInterval(interval);

x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json

@@ -2,42 +2,66 @@
   "config": {
     "id": "fq_date_histogram_1m_1441",
     "source": { "index": ["farequote-2019"], "query": { "match_all": {} } },
-    "dest": { "index": "fq_data_histogram_1m_1441" },
+    "dest": { "index": "fq_date_histogram_1m_1441" },
     "pivot": {
       "group_by": {
-        "date_histogram(@timestamp)": {
-          "date_histogram": { "field": "@timestamp", "interval": "1m" }
+        "@timestamp": {
+          "date_histogram": { "field": "@timestamp", "calendar_interval": "1m" }
         }
       },
-      "aggregations": { "avg(response)": { "avg": { "field": "responsetime" } } }
-    }
+      "aggregations": { "responsetime.avg": { "avg": { "field": "responsetime" } } }
+    },
+    "version": "8.0.0",
+    "create_time": 1564388146667,
+    "mode": "batch"
   },
   "id": "fq_date_histogram_1m_1441",
-  "state": {
-    "task_state": "stopped",
-    "indexer_state": "stopped",
-    "current_position": { "date_histogram(@timestamp)": 1549929540000 },
-    "checkpoint": 1
-  },
-  "stats": {
-    "pages_processed": 0,
-    "documents_processed": 0,
-    "documents_indexed": 0,
-    "trigger_count": 0,
-    "index_time_in_ms": 0,
-    "index_total": 0,
-    "index_failures": 0,
-    "search_time_in_ms": 0,
-    "search_total": 0,
-    "search_failures": 0
-  },
   "checkpointing": {
-    "current": {
-      "timestamp": "2019-06-28T16:09:23.539Z",
-      "timestamp_millis": 1561738163539,
-      "time_upper_bound": "2019-06-28T16:09:13.539Z",
-      "time_upper_bound_millis": 1561738153539
+    "last": {
+      "checkpoint": 1,
+      "timestamp_millis": 1564388281199
+    },
+    "next": {
+      "checkpoint": 2,
+      "indexer_state": "stopped",
+      "checkpoint_progress": {
+        "total_docs": 86274,
+        "docs_remaining": 0,
+        "percent_complete": 100
+      }
     },
     "operations_behind": 0
+  },
+  "stats": {
+    "id": "fq_date_histogram_1m_1441",
+    "task_state": "stopped",
+    "stats": {
+      "pages_processed": 16,
+      "documents_processed": 86274,
+      "documents_indexed": 7200,
+      "trigger_count": 1,
+      "index_time_in_ms": 1310,
+      "index_total": 15,
+      "index_failures": 0,
+      "search_time_in_ms": 463,
+      "search_total": 16,
+      "search_failures": 0
+    },
+    "checkpointing": {
+      "last": {
+        "checkpoint": 1,
+        "timestamp_millis": 1564388281199
+      },
+      "next": {
+        "checkpoint": 2,
+        "indexer_state": "stopped",
+        "checkpoint_progress": {
+          "total_docs": 86274,
+          "docs_remaining": 0,
+          "percent_complete": 100
+        }
+      },
+      "operations_behind": 0
+    }
   }
 }