From 75ed3a13cda366c92b99ce5f7eb0bb48652d67dc Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 29 Jul 2019 10:44:22 +0200 Subject: [PATCH 1/3] [ML] Fix stats. --- .../step_create/step_create_form.tsx | 9 +- .../data_frame_transform_list_row.json | 80 +++++++++------ .../__snapshots__/expanded_row.test.tsx.snap | 98 +++++++++---------- .../expanded_row_json_pane.test.tsx.snap | 13 ++- .../transform_list/action_delete.tsx | 2 +- .../components/transform_list/actions.tsx | 2 +- .../components/transform_list/columns.tsx | 25 +++-- .../components/transform_list/common.test.ts | 17 ++-- .../components/transform_list/common.ts | 70 ++++++++----- .../transform_list/expanded_row.tsx | 8 +- .../transform_list/transform_list.tsx | 2 +- .../transform_service/delete_transform.ts | 4 +- .../transform_service/get_transforms.ts | 17 +--- .../transform_service/start_transform.ts | 2 +- .../transform_service/stop_transform.ts | 2 +- 15 files changed, 201 insertions(+), 150 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx index 0037ffe4396e1..03056467f27cf 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { get } from 'lodash'; import React, { Fragment, SFC, useContext, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; @@ -191,7 +192,13 @@ export const StepCreateForm: SFC = React.memo( try { const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId); if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) { - const percent = Math.round(stats.transforms[0].state.progress.percent_complete); + const percent = Math.round( + get( + stats, + 'transforms[0].checkpointing.next.checkpoint_progress.percent_complete', + 0 + ) + ); setProgressPercentComplete(percent); if (percent >= 100) { clearInterval(interval); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json index f2045ea78112b..5165c693326af 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json @@ -2,42 +2,66 @@ "config": { "id": "fq_date_histogram_1m_1441", "source": { "index": ["farequote-2019"], "query": { "match_all": {} } }, - "dest": { "index": "fq_data_histogram_1m_1441" }, + "dest": { "index": "fq_date_histogram_1m_1441" }, "pivot": { "group_by": { - "date_histogram(@timestamp)": { - "date_histogram": { "field": "@timestamp", "interval": "1m" } + "@timestamp": { + "date_histogram": { "field": "@timestamp", "calendar_interval": "1m" } } }, - "aggregations": { "avg(response)": { "avg": { "field": "responsetime" } } } - } + "aggregations": { "responsetime.avg": { "avg": { "field": "responsetime" } } } + }, + "version": "8.0.0", + "create_time": 1564388146667, + "mode": "batch" }, "id": "fq_date_histogram_1m_1441", - "state": { - "task_state": "stopped", - "indexer_state": "stopped", - "current_position": { "date_histogram(@timestamp)": 1549929540000 }, - "checkpoint": 1 - }, - "stats": { - "pages_processed": 0, - "documents_processed": 0, - "documents_indexed": 0, - "trigger_count": 0, - "index_time_in_ms": 0, - "index_total": 0, - "index_failures": 0, - "search_time_in_ms": 0, - "search_total": 0, - "search_failures": 0 - }, "checkpointing": { - "current": { - "timestamp": "2019-06-28T16:09:23.539Z", - "timestamp_millis": 1561738163539, - "time_upper_bound": "2019-06-28T16:09:13.539Z", - "time_upper_bound_millis": 1561738153539 + "last": { + "checkpoint": 1, + "timestamp_millis": 1564388281199 + }, + "next": { + "checkpoint": 2, + "indexer_state": "stopped", + "checkpoint_progress": { + "total_docs": 86274, + "docs_remaining": 0, + "percent_complete": 100 + } }, "operations_behind": 0 + }, + "stats": { + "id": "fq_date_histogram_1m_1441", + "task_state": "stopped", + "stats": { + "pages_processed": 16, + "documents_processed": 86274, + "documents_indexed": 7200, + "trigger_count": 1, + "index_time_in_ms": 1310, + "index_total": 15, + "index_failures": 0, + "search_time_in_ms": 463, + "search_total": 16, + "search_failures": 0 + }, + "checkpointing": { + "last": { + "checkpoint": 1, + "timestamp_millis": 1564388281199 + }, + "next": { + "checkpoint": 2, + "indexer_state": "stopped", + "checkpoint_progress": { + "total_docs": 86274, + "docs_remaining": 0, + "percent_complete": 100 + } + }, + "operations_behind": 0 + } } } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row.test.tsx.snap b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row.test.tsx.snap index 83cc9a5148698..9063535b4445d 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row.test.tsx.snap +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row.test.tsx.snap @@ -12,20 +12,12 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "stopped", - "title": "task_state", + "description": "fq_date_histogram_1m_1441", + "title": "id", }, Object { "description": "stopped", - "title": "indexer_state", - }, - Object { - "description": "{\\"date_histogram(@timestamp)\\":1549929540000}", - "title": "current_position", - }, - Object { - "description": "1", - "title": "checkpoint", + "title": "task_state", }, ], "position": "left", @@ -34,8 +26,12 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "{\\"timestamp\\":\\"2019-06-28T16:09:23.539Z\\",\\"timestamp_millis\\":1561738163539,\\"time_upper_bound\\":\\"2019-06-28T16:09:13.539Z\\",\\"time_upper_bound_millis\\":1561738153539}", - "title": "current", + "description": "{\\"checkpoint\\":1,\\"timestamp_millis\\":1564388281199}", + "title": "last", + }, + Object { + "description": "{\\"checkpoint\\":2,\\"indexer_state\\":\\"stopped\\",\\"checkpoint_progress\\":{\\"total_docs\\":86274,\\"docs_remaining\\":0,\\"percent_complete\\":100}}", + "title": "next", }, Object { "description": "0", @@ -48,27 +44,27 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "0", + "description": "16", "title": "pages_processed", }, Object { - "description": "0", + "description": "86274", "title": "documents_processed", }, Object { - "description": "0", + "description": "7200", "title": "documents_indexed", }, Object { - "description": "0", + "description": "1", "title": "trigger_count", }, Object { - "description": "0", + "description": "1310", "title": "index_time_in_ms", }, Object { - "description": "0", + "description": "15", "title": "index_total", }, Object { @@ -76,11 +72,11 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = "title": "index_failures", }, Object { - "description": "0", + "description": "463", "title": "search_time_in_ms", }, Object { - "description": "0", + "description": "16", "title": "search_total", }, Object { @@ -109,20 +105,12 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "stopped", - "title": "task_state", + "description": "fq_date_histogram_1m_1441", + "title": "id", }, Object { "description": "stopped", - "title": "indexer_state", - }, - Object { - "description": "{\\"date_histogram(@timestamp)\\":1549929540000}", - "title": "current_position", - }, - Object { - "description": "1", - "title": "checkpoint", + "title": "task_state", }, ], "position": "left", @@ -131,8 +119,12 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "{\\"timestamp\\":\\"2019-06-28T16:09:23.539Z\\",\\"timestamp_millis\\":1561738163539,\\"time_upper_bound\\":\\"2019-06-28T16:09:13.539Z\\",\\"time_upper_bound_millis\\":1561738153539}", - "title": "current", + "description": "{\\"checkpoint\\":1,\\"timestamp_millis\\":1564388281199}", + "title": "last", + }, + Object { + "description": "{\\"checkpoint\\":2,\\"indexer_state\\":\\"stopped\\",\\"checkpoint_progress\\":{\\"total_docs\\":86274,\\"docs_remaining\\":0,\\"percent_complete\\":100}}", + "title": "next", }, Object { "description": "0", @@ -145,27 +137,27 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = Object { "items": Array [ Object { - "description": "0", + "description": "16", "title": "pages_processed", }, Object { - "description": "0", + "description": "86274", "title": "documents_processed", }, Object { - "description": "0", + "description": "7200", "title": "documents_indexed", }, Object { - "description": "0", + "description": "1", "title": "trigger_count", }, Object { - "description": "0", + "description": "1310", "title": "index_time_in_ms", }, Object { - "description": "0", + "description": "15", "title": "index_total", }, Object { @@ -173,11 +165,11 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = "title": "index_failures", }, Object { - "description": "0", + "description": "463", "title": "search_time_in_ms", }, Object { - "description": "0", + "description": "16", "title": "search_total", }, Object { @@ -198,23 +190,25 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = "content": Minimal initialization 1`] = "match_all": Object {}, }, }, + "version": "8.0.0", } } />, @@ -244,23 +239,25 @@ exports[`Data Frame: Transform List Minimal initialization 1`] = "content": Minimal initialization 1`] = "match_all": Object {}, }, }, + "version": "8.0.0", } } />, diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_json_pane.test.tsx.snap b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_json_pane.test.tsx.snap index a771d68d2fb7a..a4e25312b4c32 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_json_pane.test.tsx.snap +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_json_pane.test.tsx.snap @@ -27,25 +27,28 @@ exports[`Data Frame: Transform List Expanded Row Minimal } }, \\"dest\\": { - \\"index\\": \\"fq_data_histogram_1m_1441\\" + \\"index\\": \\"fq_date_histogram_1m_1441\\" }, \\"pivot\\": { \\"group_by\\": { - \\"date_histogram(@timestamp)\\": { + \\"@timestamp\\": { \\"date_histogram\\": { \\"field\\": \\"@timestamp\\", - \\"interval\\": \\"1m\\" + \\"calendar_interval\\": \\"1m\\" } } }, \\"aggregations\\": { - \\"avg(response)\\": { + \\"responsetime.avg\\": { \\"avg\\": { \\"field\\": \\"responsetime\\" } } } - } + }, + \\"version\\": \\"8.0.0\\", + \\"create_time\\": 1564388146667, + \\"mode\\": \\"batch\\" }" /> diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx index 0b6640aea328e..8787ef49492c5 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx @@ -28,7 +28,7 @@ interface DeleteActionProps { } export const DeleteAction: SFC = ({ item }) => { - const disabled = item.state.task_state === DATA_FRAME_TASK_STATE.STARTED; + const disabled = item.stats.task_state === DATA_FRAME_TASK_STATE.STARTED; const canDeleteDataFrame: boolean = checkPermission('canDeleteDataFrame'); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx index bcc057859b766..1aaa3df71f802 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx @@ -26,7 +26,7 @@ export const getActions = () => { { isPrimary: true, render: (item: DataFrameTransformListRow) => { - if (item.state.task_state !== DATA_FRAME_TASK_STATE.STARTED) { + if (item.stats.task_state !== DATA_FRAME_TASK_STATE.STARTED) { return ; } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx index a99aeb6839cd0..6991c06c0d0bc 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { get } from 'lodash'; import React, { Fragment } from 'react'; import { i18n } from '@kbn/i18n'; import { @@ -22,7 +23,7 @@ import { DATA_FRAME_TASK_STATE, DataFrameTransformListColumn, DataFrameTransformListRow, - DataFrameTransformState, + DataFrameTransformStats, } from './common'; import { getActions } from './actions'; @@ -33,8 +34,8 @@ enum TASK_STATE_COLOR { } export const getTaskStateBadge = ( - state: DataFrameTransformState['task_state'], - reason?: DataFrameTransformState['reason'] + state: DataFrameTransformStats['task_state'], + reason?: DataFrameTransformStats['reason'] ) => { const color = TASK_STATE_COLOR[state]; @@ -125,10 +126,10 @@ export const getColumns = ( }, { name: i18n.translate('xpack.ml.dataframe.status', { defaultMessage: 'Status' }), - sortable: (item: DataFrameTransformListRow) => item.state.task_state, + sortable: (item: DataFrameTransformListRow) => item.stats.task_state, truncateText: true, render(item: DataFrameTransformListRow) { - return getTaskStateBadge(item.state.task_state, item.state.reason); + return getTaskStateBadge(item.stats.task_state, item.stats.reason); }, width: '100px', }, @@ -146,14 +147,12 @@ export const getColumns = ( { name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }), sortable: (item: DataFrameTransformListRow) => - item.state.progress !== undefined ? item.state.progress.percent_complete : 0, + get(item, 'stats.checkpointing.next.checkpoint_progress.percent_complete', 0), truncateText: true, render(item: DataFrameTransformListRow) { - let progress = 0; - - if (item.state.progress !== undefined) { - progress = Math.round(item.state.progress.percent_complete); - } + const progress = Math.round( + get(item, 'stats.checkpointing.next.checkpoint_progress.percent_complete', 0) + ); const isBatchTransform = typeof item.config.sync === 'undefined'; @@ -174,10 +173,10 @@ export const getColumns = ( {!isBatchTransform && ( - {item.state.task_state === DATA_FRAME_TASK_STATE.STARTED && ( + {item.stats.task_state === DATA_FRAME_TASK_STATE.STARTED && ( )} - {item.state.task_state !== DATA_FRAME_TASK_STATE.STOPPED && ( + {item.stats.task_state !== DATA_FRAME_TASK_STATE.STOPPED && ( )} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts index af3ed6f373544..73483b516c7fd 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts @@ -6,22 +6,25 @@ import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; -import { DATA_FRAME_TASK_STATE, isCompletedBatchTransform } from './common'; +import { + DataFrameTransformListRow, + isCompletedBatchTransform, + DATA_FRAME_TASK_STATE, +} from './common'; describe('Data Frame: isCompletedBatchTransform()', () => { test('isCompletedBatchTransform()', () => { // check the transform config/state against the conditions // that will be used by isCompletedBatchTransform() // followed by a call to isCompletedBatchTransform() itself - expect(mockDataFrameTransformListRow.state.checkpoint === 1).toBe(true); - expect(mockDataFrameTransformListRow.sync === undefined).toBe(true); - expect(mockDataFrameTransformListRow.state.task_state === DATA_FRAME_TASK_STATE.STOPPED).toBe( - true - ); + const row = mockDataFrameTransformListRow as DataFrameTransformListRow; + expect(row.stats.checkpointing.last.checkpoint === 1).toBe(true); + expect(row.config.sync === undefined).toBe(true); + expect(row.stats.task_state === DATA_FRAME_TASK_STATE.STOPPED).toBe(true); expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(true); // adapt the mock config to resemble a non-completed transform. - mockDataFrameTransformListRow.state.checkpoint = 0; + row.stats.checkpointing.last.checkpoint = 0; expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(false); }); }); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts index 1fb90b2d60e20..bfe6b0ec32788 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts @@ -8,6 +8,12 @@ import { Dictionary } from '../../../../../../common/types/common'; import { DataFrameTransformId, DataFrameTransformPivotConfig } from '../../../../common'; +export enum DATA_FRAME_INDEXER_STATE { + FAILED = 'failed', + STARTED = 'started', + STOPPED = 'stopped', +} + export enum DATA_FRAME_TASK_STATE { FAILED = 'failed', STARTED = 'started', @@ -33,16 +39,37 @@ export interface Query { syntax: any; } -export interface DataFrameTransformState { - checkpoint: number; - current_position: Dictionary; - // indexer_state is a backend internal attribute - // and should not be considered in the UI. - indexer_state: DATA_FRAME_TASK_STATE; - progress?: { - docs_remaining: number; - percent_complete: number; - total_docs: number; +export interface DataFrameTransformStats { + id: DataFrameTransformId; + checkpointing: { + last: { + checkpoint: number; + timestamp_millis?: number; + }; + next?: { + checkpoint: number; + // indexer_state is a backend internal attribute + // and should not be considered in the UI. + indexer_state: DATA_FRAME_INDEXER_STATE; + checkpoint_progress?: { + total_docs: number; + docs_remaining: number; + percent_complete: number; + }; + }; + operations_behind: number; + }; + stats: { + documents_indexed: number; + documents_processed: number; + index_failures: number; + index_time_in_ms: number; + index_total: number; + pages_processed: number; + search_failures: number; + search_time_in_ms: number; + search_total: number; + trigger_count: number; }; reason?: string; // task_state is the attribute to check against if a transform @@ -50,23 +77,18 @@ export interface DataFrameTransformState { task_state: DATA_FRAME_TASK_STATE; } -export interface DataFrameTransformStats { - documents_indexed: number; - documents_processed: number; - index_failures: number; - index_time_in_ms: number; - index_total: number; - pages_processed: number; - search_failures: number; - search_time_in_ms: number; - search_total: number; - trigger_count: number; +export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats { + return ( + typeof arg === 'object' && + arg !== null && + {}.hasOwnProperty.call(arg, 'task_state') && + Object.values(DATA_FRAME_TASK_STATE).includes(arg.task_state) + ); } export interface DataFrameTransformListRow { id: DataFrameTransformId; checkpointing: object; - state: DataFrameTransformState; stats: DataFrameTransformStats; config: DataFrameTransformPivotConfig; } @@ -85,8 +107,8 @@ export function isCompletedBatchTransform(item: DataFrameTransformListRow) { // If `checkpoint=1`, `sync` is missing from the config and state is stopped, // then this is a completed batch data frame transform. return ( - item.state.checkpoint === 1 && + item.stats.checkpointing.last.checkpoint === 1 && item.config.sync === undefined && - item.state.task_state === DATA_FRAME_TASK_STATE.STOPPED + item.stats.task_state === DATA_FRAME_TASK_STATE.STOPPED ); } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx index 7d71429f732c0..893c1c0c350ba 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx @@ -29,9 +29,13 @@ interface Props { } export const ExpandedRow: SFC = ({ item }) => { + const stateValues = { ...item.stats }; + delete stateValues.stats; + delete stateValues.checkpointing; + const state: SectionConfig = { title: 'State', - items: Object.entries(item.state).map(s => { + items: Object.entries(stateValues).map(s => { return { title: s[0].toString(), description: getItemDescription(s[1]) }; }), position: 'left', @@ -47,7 +51,7 @@ export const ExpandedRow: SFC = ({ item }) => { const stats: SectionConfig = { title: 'Stats', - items: Object.entries(item.stats).map(s => { + items: Object.entries(item.stats.stats).map(s => { return { title: s[0].toString(), description: getItemDescription(s[1]) }; }), position: 'right', diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx index badc4eabd541d..4aaeb031fefd0 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx @@ -152,7 +152,7 @@ export const DataFrameTransformList: SFC = () => { // filter other clauses, i.e. the mode and status filters if (Array.isArray(c.value)) { // the status value is an array of string(s) e.g. ['failed', 'stopped'] - ts = transforms.filter(transform => c.value.includes(transform.state.task_state)); + ts = transforms.filter(transform => c.value.includes(transform.stats.task_state)); } else { ts = transforms.filter(transform => transform.config.mode === c.value); } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts index 3ecf5067d3fa1..ad563a6e40baa 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts @@ -17,10 +17,10 @@ import { export const deleteTransform = async (d: DataFrameTransformListRow) => { try { - if (d.state.task_state === DATA_FRAME_TASK_STATE.FAILED) { + if (d.stats.task_state === DATA_FRAME_TASK_STATE.FAILED) { await ml.dataFrame.stopDataFrameTransform( d.config.id, - d.state.task_state === DATA_FRAME_TASK_STATE.FAILED, + d.stats.task_state === DATA_FRAME_TASK_STATE.FAILED, true ); } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts index 3306009076522..81160a0aa5852 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts @@ -7,25 +7,17 @@ import { ml } from '../../../../../services/ml_api_service'; import { DataFrameTransformPivotConfig, - DataFrameTransformId, refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE, } from '../../../../common'; import { DataFrameTransformListRow, - DataFrameTransformState, DataFrameTransformStats, DATA_FRAME_MODE, + isDataFrameTransformStats, } from '../../components/transform_list/common'; -interface DataFrameTransformStateStats { - id: DataFrameTransformId; - checkpointing: object; - state: DataFrameTransformState; - stats: DataFrameTransformStats; -} - interface GetDataFrameTransformsResponse { count: number; transforms: DataFrameTransformPivotConfig[]; @@ -34,7 +26,7 @@ interface GetDataFrameTransformsResponse { interface GetDataFrameTransformsStatsResponseOk { node_failures?: object; count: number; - transforms: DataFrameTransformStateStats[]; + transforms: DataFrameTransformStats[]; } const isGetDataFrameTransformsStatsResponseOk = ( @@ -90,7 +82,7 @@ export const getTransformsFactory = ( // A newly created transform might not have corresponding stats yet. // If that's the case we just skip the transform and don't add it to the transform list yet. - if (stats === undefined) { + if (!isDataFrameTransformStats(stats)) { return reducedtableRows; } @@ -104,8 +96,7 @@ export const getTransformsFactory = ( config, id: config.id, checkpointing: stats.checkpointing, - state: stats.state, - stats: stats.stats, + stats, }); return reducedtableRows; }, diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts index e131af94a258e..0e1f7562143c0 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts @@ -19,7 +19,7 @@ export const startTransform = async (d: DataFrameTransformListRow) => { try { await ml.dataFrame.startDataFrameTransform( d.config.id, - d.state.task_state === DATA_FRAME_TASK_STATE.FAILED + d.stats.task_state === DATA_FRAME_TASK_STATE.FAILED ); toastNotifications.addSuccess( i18n.translate('xpack.ml.dataframe.transformList.startTransformSuccessMessage', { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts index c5d6b348c38a2..61550291255c1 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts @@ -19,7 +19,7 @@ export const stopTransform = async (d: DataFrameTransformListRow) => { try { await ml.dataFrame.stopDataFrameTransform( d.config.id, - d.state.task_state === DATA_FRAME_TASK_STATE.FAILED, + d.stats.task_state === DATA_FRAME_TASK_STATE.FAILED, true ); toastNotifications.addSuccess( From 34f47a7e8d10dfcc43c34802efa15a920f34b88c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 29 Jul 2019 12:28:34 +0200 Subject: [PATCH 2/3] [ML] Use idx instead of get. --- .../components/source_index_preview/expanded_row.tsx | 6 +++--- .../source_index_preview/use_source_index_data.ts | 6 +++--- .../components/step_create/step_create_form.tsx | 9 ++++----- .../components/transform_list/columns.tsx | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/expanded_row.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/expanded_row.tsx index a0fec331f2afe..f765059a9d95f 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/expanded_row.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/expanded_row.tsx @@ -4,11 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; - import React from 'react'; import { EuiBadge, EuiText } from '@elastic/eui'; +import { idx } from '@kbn/elastic-idx'; import { getSelectableFields, EsDoc } from '../../../../common'; @@ -19,7 +18,8 @@ interface ExpandedRowProps { export const ExpandedRow: React.SFC = ({ item }) => { const keys = getSelectableFields([item]); const list = keys.map(k => { - const value = get(item._source, k, ''); + // split the attribute key string and use reduce with an idx check to access nested attributes. + const value = k.split('.').reduce((obj, i) => idx(obj, _ => _[i]), item._source) || ''; return ( {k}: diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/use_source_index_data.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/use_source_index_data.ts index 1e1ecea8f62fb..c01f6d84043e6 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/use_source_index_data.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/source_index_preview/use_source_index_data.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; - import React, { useEffect, useState } from 'react'; import { SearchResponse } from 'elasticsearch'; +import { idx } from '@kbn/elastic-idx'; + import { IndexPattern } from 'ui/index_patterns'; import { ml } from '../../../../../services/ml_api_service'; @@ -82,7 +82,7 @@ export const useSourceIndexData = ( [key: string]: any; }; flattenedFields.forEach(ff => { - item[ff] = get(doc._source, ff); + item[ff] = idx(doc._source, _ => _[ff]); if (item[ff] === undefined) { item[ff] = doc._source[`"${ff}"`]; } diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx index 03056467f27cf..7ea15673fb18a 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; import React, { Fragment, SFC, useContext, useEffect, useState } from 'react'; +import { idx } from '@kbn/elastic-idx'; import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; @@ -193,11 +193,10 @@ export const StepCreateForm: SFC = React.memo( const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId); if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) { const percent = Math.round( - get( + idx( stats, - 'transforms[0].checkpointing.next.checkpoint_progress.percent_complete', - 0 - ) + _ => _.transforms[0].checkpointing.next.checkpoint_progress.percent_complete + ) || 0 ); setProgressPercentComplete(percent); if (percent >= 100) { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx index 6991c06c0d0bc..bafbd4c10de31 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; import React, { Fragment } from 'react'; +import { idx } from '@kbn/elastic-idx'; import { i18n } from '@kbn/i18n'; import { EuiBadge, @@ -147,11 +147,11 @@ export const getColumns = ( { name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }), sortable: (item: DataFrameTransformListRow) => - get(item, 'stats.checkpointing.next.checkpoint_progress.percent_complete', 0), + idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete) || 0, truncateText: true, render(item: DataFrameTransformListRow) { const progress = Math.round( - get(item, 'stats.checkpointing.next.checkpoint_progress.percent_complete', 0) + idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete) || 0 ); const isBatchTransform = typeof item.config.sync === 'undefined'; From 40703f0a81dc490e16d1371a49131eb6d1ca100c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 29 Jul 2019 13:12:28 +0200 Subject: [PATCH 3/3] [ML] Fix details pane width. --- .../expanded_row_details_pane.test.tsx.snap | 16 ++++++++++++++-- .../transform_list/expanded_row_details_pane.tsx | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_details_pane.test.tsx.snap b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_details_pane.test.tsx.snap index 2f2048daa6b01..88f1d20f119e8 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_details_pane.test.tsx.snap +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__snapshots__/expanded_row_details_pane.test.tsx.snap @@ -2,7 +2,13 @@ exports[`Data Frame: Job List Expanded Row Minimal initialization 1`] = ` - + @@ -21,7 +27,13 @@ exports[`Data Frame: Job List Expanded Row Minimal in } /> - + `; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_details_pane.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_details_pane.tsx index 4beb0f2cd1639..7ba05a5fe41ec 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_details_pane.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_details_pane.tsx @@ -51,7 +51,7 @@ interface ExpandedRowDetailsPaneProps { export const ExpandedRowDetailsPane: SFC = ({ sections }) => { return ( - + {sections .filter(s => s.position === 'left') .map(s => ( @@ -61,7 +61,7 @@ export const ExpandedRowDetailsPane: SFC = ({ secti ))} - + {sections .filter(s => s.position === 'right') .map(s => (