From 665fbc799b23b40b8b67b5ef873400e9998adbc7 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 10 Sep 2019 01:28:33 -0700 Subject: [PATCH] [ML] Data frame transform: Fix progress in wizard create step. (#45116) Fixes a regression where the progress bar in the transform wizard would return to 0% after a transform finishes. The actual bugfix is just the code in 6033a4c to make sure the code in the wizard uses the same function getTransformProgress() like the transform management page. Also includes a fix for sorting on the transform list table related to progress. The rest of the diff is a) moving types and utility functions previously only used within the transform management page to the common shared code of data frame transforms so it's available for the wizard without cross-deep-importing (717d894) and b) adding tests with updated mock data (f14e231). --- .../data_frame_transform_list_row.json | 0 .../__mocks__/data_frame_transform_stats.json | 130 ++++++++++++++++++ .../ml/public/data_frame/common/index.ts | 9 ++ .../data_frame/common/transform_list.ts | 23 ++++ .../data_frame/common/transform_stats.test.ts | 40 ++++++ .../data_frame/common/transform_stats.ts | 94 +++++++++++++ .../step_create/step_create_form.tsx | 19 +-- .../transform_list/action_delete.test.tsx | 4 +- .../transform_list/action_delete.tsx | 2 +- .../transform_list/action_start.test.tsx | 4 +- .../transform_list/action_start.tsx | 2 +- .../transform_list/action_stop.test.tsx | 4 +- .../components/transform_list/action_stop.tsx | 2 +- .../components/transform_list/actions.tsx | 2 +- .../components/transform_list/columns.tsx | 18 +-- .../components/transform_list/common.test.ts | 4 +- .../components/transform_list/common.ts | 107 +------------- .../transform_list/expanded_row.test.tsx | 4 +- .../transform_list/expanded_row.tsx | 2 +- .../expanded_row_json_pane.test.tsx | 2 +- .../transform_list/transform_list.tsx | 25 ++-- .../transform_list/transforms_stats_bar.tsx | 6 +- .../pages/transform_management/page.tsx | 3 +- .../transform_service/delete_transform.ts | 5 +- .../transform_service/get_transforms.ts | 11 +- .../transform_service/start_transform.ts | 7 +- .../transform_service/stop_transform.ts | 7 +- .../ml/server/models/data_frame/transforms.ts | 2 +- 28 files changed, 371 insertions(+), 167 deletions(-) rename x-pack/legacy/plugins/ml/public/data_frame/{pages/transform_management/components/transform_list => common}/__mocks__/data_frame_transform_list_row.json (100%) create mode 100644 x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json create mode 100644 x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts create mode 100644 x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts create mode 100644 x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json b/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_list_row.json similarity index 100% rename from x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/__mocks__/data_frame_transform_list_row.json rename to x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_list_row.json diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json b/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json new file mode 100644 index 0000000000000..0e11cf3d75bb9 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json @@ -0,0 +1,130 @@ +{ + "count" : 4, + "transforms" : [ + { + "id" : "transform-created", + "state" : "stopped", + "stats" : { + "pages_processed" : 0, + "documents_processed" : 0, + "documents_indexed" : 0, + "trigger_count" : 0, + "index_time_in_ms" : 0, + "index_total" : 0, + "index_failures" : 0, + "search_time_in_ms" : 0, + "search_total" : 0, + "search_failures" : 0, + "exponential_avg_checkpoint_duration_ms" : 0.0, + "exponential_avg_documents_indexed" : 0.0, + "exponential_avg_documents_processed" : 0.0 + }, + "checkpointing" : { + "last" : { + "checkpoint" : 0 + }, + "operations_behind" : 0 + } + }, + { + "id" : "transform-created-started-stopped", + "state" : "stopped", + "stats" : { + "pages_processed" : 2, + "documents_processed" : 2864381, + "documents_indexed" : 1000, + "trigger_count" : 1, + "index_time_in_ms" : 52, + "index_total" : 2, + "index_failures" : 0, + "search_time_in_ms" : 1859, + "search_total" : 3, + "search_failures" : 0, + "exponential_avg_checkpoint_duration_ms" : 0.0, + "exponential_avg_documents_indexed" : 0.0, + "exponential_avg_documents_processed" : 0.0 + }, + "checkpointing" : { + "last" : { + "checkpoint" : 0 + }, + "operations_behind" : 21139495 + } + }, + { + "id" : "transform-running", + "state" : "indexing", + "node" : { + "id" : "xxx", + "name" : "node", + "ephemeral_id" : "xxx", + "transport_address" : "127.0.0.1:9300", + "attributes" : { } + }, + "stats" : { + "pages_processed" : 3, + "documents_processed" : 4457902, + "documents_indexed" : 1500, + "trigger_count" : 2, + "index_time_in_ms" : 101, + "index_total" : 3, + "index_failures" : 0, + "search_time_in_ms" : 1909, + "search_total" : 4, + "search_failures" : 0, + "exponential_avg_checkpoint_duration_ms" : 0.0, + "exponential_avg_documents_indexed" : 0.0, + "exponential_avg_documents_processed" : 0.0 + }, + "checkpointing" : { + "last" : { + "checkpoint" : 0 + }, + "next" : { + "checkpoint" : 1, + "position" : { + "indexer_position" : { + "@timestamp" : 1551282000000, + "agent.version" : "7.0.0" + } + }, + "checkpoint_progress" : { + "docs_remaining" : 16681594, + "total_docs" : 21139496, + "percent_complete" : 21.08802404749858, + "docs_indexed" : 1500, + "docs_processed" : 4457902 + }, + "timestamp_millis" : 1568021396011 + }, + "operations_behind" : 21139495 + } + }, + { + "id" : "transform-completed", + "state" : "stopped", + "stats" : { + "pages_processed" : 22, + "documents_processed" : 21139496, + "documents_indexed" : 10154, + "trigger_count" : 2, + "index_time_in_ms" : 530, + "index_total" : 21, + "index_failures" : 0, + "search_time_in_ms" : 8566, + "search_total" : 23, + "search_failures" : 0, + "exponential_avg_checkpoint_duration_ms" : 0.0, + "exponential_avg_documents_indexed" : 0.0, + "exponential_avg_documents_processed" : 0.0 + }, + "checkpointing" : { + "last" : { + "checkpoint" : 1, + "timestamp_millis" : 1568021396011 + }, + "operations_behind" : 0 + } + } + ] +} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts index eeb29633c844b..8b33eb666c738 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts @@ -28,6 +28,15 @@ export { IndexPattern, REFRESH_TRANSFORM_LIST_STATE, } from './transform'; +export { DATA_FRAME_TRANSFORM_LIST_COLUMN, DataFrameTransformListRow } from './transform_list'; +export { + getTransformProgress, + isCompletedBatchTransform, + isDataFrameTransformStats, + DataFrameTransformStats, + DATA_FRAME_MODE, + DATA_FRAME_TRANSFORM_STATE, +} from './transform_stats'; export { moveToDataFrameTransformList, moveToDataFrameWizard, moveToDiscover } from './navigation'; export { getEsAggFromAggConfig, diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts new file mode 100644 index 0000000000000..0f84acf5d72cf --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DataFrameTransformId, DataFrameTransformPivotConfig } from './transform'; +import { DataFrameTransformStats } from './transform_stats'; + +// Used to pass on attribute names to table columns +export enum DATA_FRAME_TRANSFORM_LIST_COLUMN { + CONFIG_DEST_INDEX = 'config.dest.index', + CONFIG_SOURCE_INDEX = 'config.source.index', + DESCRIPTION = 'config.description', + ID = 'id', +} + +export interface DataFrameTransformListRow { + id: DataFrameTransformId; + config: DataFrameTransformPivotConfig; + mode?: string; // added property on client side to allow filtering by this field + stats: DataFrameTransformStats; +} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts new file mode 100644 index 0000000000000..a8da8bc90b43d --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import mockDataFrameTransformStats from './__mocks__/data_frame_transform_stats.json'; + +import { DataFrameTransformListRow } from './transform_list'; +import { getTransformProgress, isCompletedBatchTransform } from './transform_stats'; + +const getRow = (statsId: string) => { + return { + ...(mockDataFrameTransformListRow as DataFrameTransformListRow), + stats: { + ...mockDataFrameTransformStats.transforms.find( + (stats: DataFrameTransformListRow['stats']) => stats.id === statsId + ), + }, + }; +}; + +describe('Data Frame: Transform stats.', () => { + test('getTransformProgress()', () => { + // At the moment, any kind of stopped jobs don't include progress information. + // We cannot infer progress for now from an unfinished job that has been stopped for now. + expect(getTransformProgress(getRow('transform-created'))).toBe(undefined); + expect(getTransformProgress(getRow('transform-created-started-stopped'))).toBe(undefined); + expect(getTransformProgress(getRow('transform-running'))).toBe(21); + expect(getTransformProgress(getRow('transform-completed'))).toBe(100); + }); + + test('isCompletedBatchTransform()', () => { + expect(isCompletedBatchTransform(getRow('transform-created'))).toBe(false); + expect(isCompletedBatchTransform(getRow('transform-created-started-stopped'))).toBe(false); + expect(isCompletedBatchTransform(getRow('transform-running'))).toBe(false); + expect(isCompletedBatchTransform(getRow('transform-completed'))).toBe(true); + }); +}); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts new file mode 100644 index 0000000000000..303a953c0d3c8 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { idx } from '@kbn/elastic-idx'; + +import { DataFrameTransformId } from './transform'; +import { DataFrameTransformListRow } from './transform_list'; + +// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243 +export enum DATA_FRAME_TRANSFORM_STATE { + ABORTING = 'aborting', + FAILED = 'failed', + INDEXING = 'indexing', + STARTED = 'started', + STOPPED = 'stopped', + STOPPING = 'stopping', +} + +export enum DATA_FRAME_MODE { + BATCH = 'batch', + CONTINUOUS = 'continuous', +} + +export interface DataFrameTransformStats { + id: DataFrameTransformId; + checkpointing: { + last: { + checkpoint: number; + timestamp_millis?: number; + }; + next?: { + checkpoint: number; + checkpoint_progress?: { + total_docs: number; + docs_remaining: number; + percent_complete: number; + }; + }; + operations_behind: number; + }; + node?: { + id: string; + name: string; + ephemeral_id: string; + transport_address: string; + attributes: Record; + }; + stats: { + documents_indexed: number; + documents_processed: number; + index_failures: number; + index_time_in_ms: number; + index_total: number; + pages_processed: number; + search_failures: number; + search_time_in_ms: number; + search_total: number; + trigger_count: number; + }; + reason?: string; + state: DATA_FRAME_TRANSFORM_STATE; +} + +export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats { + return ( + typeof arg === 'object' && + arg !== null && + {}.hasOwnProperty.call(arg, 'state') && + Object.values(DATA_FRAME_TRANSFORM_STATE).includes(arg.state) + ); +} + +export function getTransformProgress(item: DataFrameTransformListRow) { + if (isCompletedBatchTransform(item)) { + return 100; + } + + const progress = idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete); + + return progress !== undefined ? Math.round(progress) : undefined; +} + +export function isCompletedBatchTransform(item: DataFrameTransformListRow) { + // If `checkpoint=1`, `sync` is missing from the config and state is stopped, + // then this is a completed batch data frame transform. + return ( + item.stats.checkpointing.last.checkpoint === 1 && + item.config.sync === undefined && + item.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED + ); +} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx index af26e99b3af99..1ce9f5749fc2f 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx @@ -5,7 +5,6 @@ */ import React, { Fragment, SFC, useEffect, useState } from 'react'; -import { idx } from '@kbn/elastic-idx'; import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; @@ -35,7 +34,11 @@ import { useKibanaContext } from '../../../../../contexts/kibana/use_kibana_cont import { useUiChromeContext } from '../../../../../contexts/ui/use_ui_chrome_context'; import { PROGRESS_JOBS_REFRESH_INTERVAL_MS } from '../../../../../../common/constants/jobs_list'; -import { moveToDataFrameTransformList, moveToDiscover } from '../../../../common'; +import { + getTransformProgress, + moveToDataFrameTransformList, + moveToDiscover, +} from '../../../../common'; export interface StepDetailsExposedState { created: boolean; @@ -198,12 +201,12 @@ export const StepCreateForm: SFC = React.memo( try { const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId); if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) { - const percent = Math.round( - idx( - stats, - _ => _.transforms[0].checkpointing.next.checkpoint_progress.percent_complete - ) || 0 - ); + const percent = + getTransformProgress({ + id: transformConfig.id, + config: transformConfig, + stats: stats.transforms[0], + }) || 0; setProgressPercentComplete(percent); if (percent >= 100) { clearInterval(interval); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx index edfc3af601c86..b94a787055eb6 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx @@ -7,10 +7,10 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { DataFrameTransformListRow } from './common'; +import { DataFrameTransformListRow } from '../../../../common'; import { DeleteAction } from './action_delete'; -import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; describe('Data Frame: Transform List Actions ', () => { test('Minimal initialization', () => { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx index 59bc06dc23ac7..3bf660d2bdd37 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx @@ -21,7 +21,7 @@ import { createPermissionFailureMessage, } from '../../../../../privilege/check_privilege'; -import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common'; +import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common'; interface DeleteActionProps { items: DataFrameTransformListRow[]; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx index 538739670dddd..17bfe36366f7e 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx @@ -7,10 +7,10 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { DataFrameTransformListRow } from './common'; +import { DataFrameTransformListRow } from '../../../../common'; import { StartAction } from './action_start'; -import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; describe('Data Frame: Transform List Actions ', () => { test('Minimal initialization', () => { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx index 577b754851542..224ce99ac7ca0 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx @@ -25,7 +25,7 @@ import { DataFrameTransformListRow, isCompletedBatchTransform, DATA_FRAME_TRANSFORM_STATE, -} from './common'; +} from '../../../../common'; interface StartActionProps { items: DataFrameTransformListRow[]; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.test.tsx index 92b8913e11a4b..b2ff630c54343 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.test.tsx @@ -7,10 +7,10 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { DataFrameTransformListRow } from './common'; +import { DataFrameTransformListRow } from '../../../../common'; import { StopAction } from './action_stop'; -import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; describe('Data Frame: Transform List Actions ', () => { test('Minimal initialization', () => { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.tsx index 41db6262afc53..fc069c0df21d6 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_stop.tsx @@ -8,7 +8,7 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiButtonEmpty, EuiToolTip } from '@elastic/eui'; -import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common'; +import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common'; import { checkPermission, createPermissionFailureMessage, diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx index 3f21ca3b85516..0bce8ec6bb3de 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/actions.tsx @@ -5,7 +5,7 @@ */ import React from 'react'; -import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common'; +import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common'; import { StartAction } from './action_start'; import { StopAction } from './action_stop'; import { DeleteAction } from './action_delete'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx index 1e18a41b50c1d..f22cfc091a948 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/columns.tsx @@ -24,14 +24,14 @@ import { FieldDataColumnType, } from '../../../../../../common/types/eui/in_memory_table'; -import { DataFrameTransformId } from '../../../../common'; import { getTransformProgress, - DATA_FRAME_TRANSFORM_STATE, - DataFrameTransformListColumn, + DataFrameTransformId, DataFrameTransformListRow, DataFrameTransformStats, -} from './common'; + DATA_FRAME_TRANSFORM_LIST_COLUMN, + DATA_FRAME_TRANSFORM_STATE, +} from '../../../../common'; import { getActions } from './actions'; enum STATE_COLOR { @@ -120,25 +120,25 @@ export const getColumns = ( ), }, { - field: DataFrameTransformListColumn.id, + field: DATA_FRAME_TRANSFORM_LIST_COLUMN.ID, name: 'ID', sortable: true, truncateText: true, }, { - field: DataFrameTransformListColumn.description, + field: DATA_FRAME_TRANSFORM_LIST_COLUMN.DESCRIPTION, name: i18n.translate('xpack.ml.dataframe.description', { defaultMessage: 'Description' }), sortable: true, truncateText: true, }, { - field: DataFrameTransformListColumn.configSourceIndex, + field: DATA_FRAME_TRANSFORM_LIST_COLUMN.CONFIG_SOURCE_INDEX, name: i18n.translate('xpack.ml.dataframe.sourceIndex', { defaultMessage: 'Source index' }), sortable: true, truncateText: true, }, { - field: DataFrameTransformListColumn.configDestIndex, + field: DATA_FRAME_TRANSFORM_LIST_COLUMN.CONFIG_DEST_INDEX, name: i18n.translate('xpack.ml.dataframe.destinationIndex', { defaultMessage: 'Destination index', }), @@ -167,7 +167,7 @@ export const getColumns = ( }, { name: i18n.translate('xpack.ml.dataframe.progress', { defaultMessage: 'Progress' }), - sortable: getTransformProgress || 0, + sortable: (item: DataFrameTransformListRow) => getTransformProgress(item) || 0, truncateText: true, render(item: DataFrameTransformListRow) { const progress = getTransformProgress(item); diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts index 38ac75d46ca94..8be5bf36b635f 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.test.ts @@ -4,13 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import mockDataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; import { DataFrameTransformListRow, isCompletedBatchTransform, DATA_FRAME_TRANSFORM_STATE, -} from './common'; +} from '../../../../common'; describe('Data Frame: isCompletedBatchTransform()', () => { test('isCompletedBatchTransform()', () => { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts index 988daea8046ad..af06276227c90 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/common.ts @@ -4,26 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { idx } from '@kbn/elastic-idx'; - -import { Dictionary } from '../../../../../../common/types/common'; - -import { DataFrameTransformId, DataFrameTransformPivotConfig } from '../../../../common'; - -// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243 -export enum DATA_FRAME_TRANSFORM_STATE { - ABORTING = 'aborting', - FAILED = 'failed', - INDEXING = 'indexing', - STARTED = 'started', - STOPPED = 'stopped', - STOPPING = 'stopping', -} - -export enum DATA_FRAME_MODE { - BATCH = 'batch', - CONTINUOUS = 'continuous', -} +import { DataFrameTransformId, DATA_FRAME_TRANSFORM_STATE } from '../../../../common'; export interface Clause { type: string; @@ -39,72 +20,6 @@ export interface Query { syntax: any; } -export interface DataFrameTransformStats { - id: DataFrameTransformId; - checkpointing: { - last: { - checkpoint: number; - timestamp_millis?: number; - }; - next?: { - checkpoint: number; - checkpoint_progress?: { - total_docs: number; - docs_remaining: number; - percent_complete: number; - }; - }; - operations_behind: number; - }; - node?: { - id: string; - name: string; - ephemeral_id: string; - transport_address: string; - attributes: Record; - }; - stats: { - documents_indexed: number; - documents_processed: number; - index_failures: number; - index_time_in_ms: number; - index_total: number; - pages_processed: number; - search_failures: number; - search_time_in_ms: number; - search_total: number; - trigger_count: number; - }; - reason?: string; - state: DATA_FRAME_TRANSFORM_STATE; -} - -export function getTransformProgress(item: DataFrameTransformListRow) { - if (isCompletedBatchTransform(item)) { - return 100; - } - - const progress = idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete); - - return progress !== undefined ? Math.round(progress) : undefined; -} - -export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats { - return ( - typeof arg === 'object' && - arg !== null && - {}.hasOwnProperty.call(arg, 'state') && - Object.values(DATA_FRAME_TRANSFORM_STATE).includes(arg.state) - ); -} - -export interface DataFrameTransformListRow { - id: DataFrameTransformId; - config: DataFrameTransformPivotConfig; - mode?: string; // added property on client side to allow filtering by this field - stats: DataFrameTransformStats; -} - export interface DataFrameTransformEndpointRequest { id: DataFrameTransformId; state?: DATA_FRAME_TRANSFORM_STATE; @@ -119,22 +34,4 @@ export interface DataFrameTransformEndpointResult { [key: string]: ResultData; } -// Used to pass on attribute names to table columns -export enum DataFrameTransformListColumn { - configDestIndex = 'config.dest.index', - configSourceIndex = 'config.source.index', - description = 'config.description', - id = 'id', -} - -export type ItemIdToExpandedRowMap = Dictionary; - -export function isCompletedBatchTransform(item: DataFrameTransformListRow) { - // If `checkpoint=1`, `sync` is missing from the config and state is stopped, - // then this is a completed batch data frame transform. - return ( - item.stats.checkpointing.last.checkpoint === 1 && - item.config.sync === undefined && - item.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED - ); -} +export type ItemIdToExpandedRowMap = Record; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.test.tsx index 62ee2a268e31a..2d49299438447 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.test.tsx @@ -8,10 +8,10 @@ import { shallow } from 'enzyme'; import React from 'react'; import moment from 'moment-timezone'; -import { DataFrameTransformListRow } from './common'; +import { DataFrameTransformListRow } from '../../../../common'; import { ExpandedRow } from './expanded_row'; -import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; describe('Data Frame: Transform List ', () => { // Set timezone to US/Eastern for consistent test results. diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx index 4079980ede6d4..ea894b7a2c720 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import { formatHumanReadableDateTimeSeconds } from '../../../../../util/date_utils'; -import { DataFrameTransformListRow } from './common'; +import { DataFrameTransformListRow } from '../../../../common'; import { ExpandedRowDetailsPane, SectionConfig } from './expanded_row_details_pane'; import { ExpandedRowJsonPane } from './expanded_row_json_pane'; import { ExpandedRowMessagesPane } from './expanded_row_messages_pane'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_json_pane.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_json_pane.test.tsx index 1303415b5d80f..6dc327caa419c 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_json_pane.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/expanded_row_json_pane.test.tsx @@ -7,7 +7,7 @@ import { shallow } from 'enzyme'; import React from 'react'; -import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json'; +import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json'; import { ExpandedRowJsonPane } from './expanded_row_json_pane'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx index 0eb51ddfa7c20..838b9678978cb 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transform_list.tsx @@ -24,22 +24,21 @@ import { SORT_DIRECTION, } from '../../../../../../common/types/eui/in_memory_table'; -import { DataFrameTransformId, moveToDataFrameWizard } from '../../../../common'; +import { + DataFrameTransformId, + DataFrameTransformListRow, + moveToDataFrameWizard, + DATA_FRAME_MODE, + DATA_FRAME_TRANSFORM_LIST_COLUMN, + DATA_FRAME_TRANSFORM_STATE, +} from '../../../../common'; import { checkPermission } from '../../../../../privilege/check_privilege'; import { getTaskStateBadge } from './columns'; import { DeleteAction } from './action_delete'; import { StartAction } from './action_start'; import { StopAction } from './action_stop'; -import { - DataFrameTransformListColumn, - DataFrameTransformListRow, - ItemIdToExpandedRowMap, - DATA_FRAME_TRANSFORM_STATE, - DATA_FRAME_MODE, - Query, - Clause, -} from './common'; +import { ItemIdToExpandedRowMap, Query, Clause } from './common'; import { getColumns } from './columns'; import { ExpandedRow } from './expanded_row'; import { ProgressBar, TransformTable } from './transform_table'; @@ -95,7 +94,7 @@ export const DataFrameTransformList: SFC = ({ const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DataFrameTransformListColumn.id); + const [sortField, setSortField] = useState(DATA_FRAME_TRANSFORM_LIST_COLUMN.ID); const [sortDirection, setSortDirection] = useState(SORT_DIRECTION.ASC); const disabled = @@ -341,7 +340,7 @@ export const DataFrameTransformList: SFC = ({ const onTableChange = ({ page = { index: 0, size: 10 }, - sort = { field: DataFrameTransformListColumn.id, direction: SORT_DIRECTION.ASC }, + sort = { field: DATA_FRAME_TRANSFORM_LIST_COLUMN.ID, direction: SORT_DIRECTION.ASC }, }: OnTableChangeArg) => { const { index, size } = page; setPageIndex(index); @@ -368,7 +367,7 @@ export const DataFrameTransformList: SFC = ({ isExpandable={true} isSelectable={false} items={filterActive ? filteredTransforms : transforms} - itemId={DataFrameTransformListColumn.id} + itemId={DATA_FRAME_TRANSFORM_LIST_COLUMN.ID} itemIdToExpandedRowMap={itemIdToExpandedRowMap} onTableChange={onTableChange} pagination={pagination} diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transforms_stats_bar.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transforms_stats_bar.tsx index 4d6a4a7669274..823a2e4c90b83 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transforms_stats_bar.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/transforms_stats_bar.tsx @@ -7,7 +7,11 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { StatsBar, TransformStatsBarStats } from '../../../../../components/stats_bar'; -import { DATA_FRAME_TRANSFORM_STATE, DATA_FRAME_MODE, DataFrameTransformListRow } from './common'; +import { + DATA_FRAME_TRANSFORM_STATE, + DATA_FRAME_MODE, + DataFrameTransformListRow, +} from '../../../../common'; function createTranformStats(transformsList: DataFrameTransformListRow[]) { const transformStats = { diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/page.tsx b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/page.tsx index 3f00a37484808..45b282bbf8355 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/page.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/page.tsx @@ -24,8 +24,7 @@ import { } from '@elastic/eui'; import { NavigationMenu } from '../../../components/navigation_menu/navigation_menu'; -import { useRefreshTransformList } from '../../common'; -import { DataFrameTransformListRow } from './components/transform_list/common'; +import { useRefreshTransformList, DataFrameTransformListRow } from '../../common'; import { CreateTransformButton } from './components/create_transform_button'; import { DataFrameTransformList } from './components/transform_list'; import { RefreshTransformListButton } from './components/refresh_transform_list_button'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts index b661890330b16..71cc42a126521 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/delete_transform.ts @@ -7,9 +7,12 @@ import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; import { ml } from '../../../../../services/ml_api_service'; -import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../../../../common'; import { DataFrameTransformListRow, + refreshTransformList$, + REFRESH_TRANSFORM_LIST_STATE, +} from '../../../../common'; +import { DataFrameTransformEndpointRequest, DataFrameTransformEndpointResult, } from '../../components/transform_list/common'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts index e6245d148e7d7..34ee8f0797cfb 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/get_transforms.ts @@ -5,18 +5,15 @@ */ import { ml } from '../../../../../services/ml_api_service'; -import { - DataFrameTransformPivotConfig, - refreshTransformList$, - REFRESH_TRANSFORM_LIST_STATE, -} from '../../../../common'; - import { DataFrameTransformListRow, DataFrameTransformStats, DATA_FRAME_MODE, isDataFrameTransformStats, -} from '../../components/transform_list/common'; + DataFrameTransformPivotConfig, + refreshTransformList$, + REFRESH_TRANSFORM_LIST_STATE, +} from '../../../../common'; interface GetDataFrameTransformsResponse { count: number; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts index 44bded99c6c31..d121e3db3b2fd 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/start_transform.ts @@ -8,10 +8,13 @@ import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; import { ml } from '../../../../../services/ml_api_service'; -import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../../../../common'; - import { DataFrameTransformListRow, + refreshTransformList$, + REFRESH_TRANSFORM_LIST_STATE, +} from '../../../../common'; + +import { DataFrameTransformEndpointRequest, DataFrameTransformEndpointResult, } from '../../components/transform_list/common'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts index 717d8841335bb..5050109d318ea 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/services/transform_service/stop_transform.ts @@ -8,10 +8,13 @@ import { i18n } from '@kbn/i18n'; import { toastNotifications } from 'ui/notify'; import { ml } from '../../../../../services/ml_api_service'; -import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../../../../common'; - import { DataFrameTransformListRow, + refreshTransformList$, + REFRESH_TRANSFORM_LIST_STATE, +} from '../../../../common'; + +import { DataFrameTransformEndpointRequest, DataFrameTransformEndpointResult, } from '../../components/transform_list/common'; diff --git a/x-pack/legacy/plugins/ml/server/models/data_frame/transforms.ts b/x-pack/legacy/plugins/ml/server/models/data_frame/transforms.ts index 031c87802a5da..392fb4191eba0 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_frame/transforms.ts +++ b/x-pack/legacy/plugins/ml/server/models/data_frame/transforms.ts @@ -5,8 +5,8 @@ */ import { callWithRequestType } from '../../../common/types/kibana'; +import { DATA_FRAME_TRANSFORM_STATE } from '../../../public/data_frame/common'; import { - DATA_FRAME_TRANSFORM_STATE, DataFrameTransformEndpointRequest, DataFrameTransformEndpointResult, } from '../../../public/data_frame/pages/transform_management/components/transform_list/common';