[7.x] [ML] Anomaly detection job custom_settings improvements (#102099)

x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts

@@ -36,6 +36,7 @@ export interface MlSummaryJob {
   earliestStartTimestampMs?: number;
   awaitingNodeAssignment: boolean;
   alertingRules?: MlAnomalyDetectionAlertRule[];
+  jobTags: Record<string, string>;
 }
 
 export interface AuditMessage {

x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/extract_job_details.js

@@ -26,19 +26,33 @@ export function extractJobDetails(job, basePath, refreshJobList) {
     items: filterObjects(job, true).map(formatValues),
   };
 
+  const { job_tags: tags, custom_urls: urls, ...settings } = job.custom_settings ?? {};
   const customUrl = {
     id: 'customUrl',
     title: i18n.translate('xpack.ml.jobsList.jobDetails.customUrlsTitle', {
       defaultMessage: 'Custom URLs',
     }),
     position: 'right',
-    items: [],
+    items: urls ? urls.map((cu) => [cu.url_name, cu.url_value, cu.time_range]) : [],
+  };
+
+  const customSettings = {
+    id: 'analysisConfig',
+    title: i18n.translate('xpack.ml.jobsList.jobDetails.customSettingsTitle', {
+      defaultMessage: 'Custom settings',
+    }),
+    position: 'right',
+    items: settings ? filterObjects(settings, true, true) : [],
+  };
+
+  const jobTags = {
+    id: 'analysisConfig',
+    title: i18n.translate('xpack.ml.jobsList.jobDetails.jobTagsTitle', {
+      defaultMessage: 'Job tags',
+    }),
+    position: 'right',
+    items: tags ? filterObjects(tags) : [],
   };
-  if (job.custom_settings && job.custom_settings.custom_urls) {
-    customUrl.items.push(
-      ...job.custom_settings.custom_urls.map((cu) => [cu.url_name, cu.url_value, cu.time_range])
-    );
-  }
 
   const node = {
     id: 'node',
@@ -213,6 +227,8 @@ export function extractJobDetails(job, basePath, refreshJobList) {
     analysisConfig,
     analysisLimits,
     dataDescription,
+    customSettings,
+    jobTags,
     datafeed,
     counts,
     modelSizeStats,

x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js

@@ -41,7 +41,7 @@ export class JobDetailsUI extends Component {
   }
 
   render() {
-    const { job } = this.state;
+    const job = this.state.job ?? this.props.job;
     const {
       services: {
         http: { basePath },
@@ -67,6 +67,8 @@ export class JobDetailsUI extends Component {
         analysisConfig,
         analysisLimits,
         dataDescription,
+        customSettings,
+        jobTags,
         datafeed,
         counts,
         modelSizeStats,
@@ -85,7 +87,7 @@ export class JobDetailsUI extends Component {
           content: (
             <JobDetailsPane
               data-test-subj="mlJobDetails-job-settings"
-              sections={[general, customUrl, node, calendars, alertRules]}
+              sections={[general, customSettings, customUrl, jobTags, node, calendars, alertRules]}
             />
           ),
           time: job.open_time,

x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js

@@ -18,7 +18,7 @@ import {
   EuiSpacer,
   EuiTitle,
 } from '@elastic/eui';
-import { isEqual } from 'lodash';
+import { isEqual, debounce } from 'lodash';
 
 import { ml } from '../../../../services/ml_api_service';
 import { checkForAutoStartDatafeed, filterJobs, loadFullJob } from '../utils';
@@ -43,6 +43,11 @@ import { JobListMlAnomalyAlertFlyout } from '../../../../../alerting/ml_alerting
 
 let deletingJobsRefreshTimeout = null;
 
+const filterJobsDebounce = debounce((jobsSummaryList, filterClauses, callback) => {
+  const ss = filterJobs(jobsSummaryList, filterClauses);
+  callback(ss);
+}, 500);
+
 // 'isManagementTable' bool prop to determine when to configure table for use in Kibana management page
 export class JobsListView extends Component {
   constructor(props) {
@@ -221,7 +226,7 @@ export class JobsListView extends Component {
 
   refreshSelectedJobs() {
     const selectedJobsIds = this.state.selectedJobs.map((j) => j.id);
-    const filteredJobIds = this.state.filteredJobsSummaryList.map((j) => j.id);
+    const filteredJobIds = (this.state.filteredJobsSummaryList ?? []).map((j) => j.id);
 
     // refresh the jobs stored as selected
     // only select those which are also in the filtered list
@@ -232,9 +237,17 @@ export class JobsListView extends Component {
     this.setState({ selectedJobs });
   }
 
-  setFilters = (query) => {
-    const filterClauses = (query && query.ast && query.ast.clauses) || [];
-    const filteredJobsSummaryList = filterJobs(this.state.jobsSummaryList, filterClauses);
+  setFilters = async (query) => {
+    if (query === null) {
+      this.setState(
+        { filteredJobsSummaryList: this.state.jobsSummaryList, filterClauses: [] },
+        () => {
+          this.refreshSelectedJobs();
+        }
+      );
+
+      return;
+    }
 
     this.props.onJobsViewStateUpdate(
       {
@@ -244,11 +257,30 @@ export class JobsListView extends Component {
       this._isFiltersSet === false
     );
 
-    this._isFiltersSet = true;
+    const filterClauses = (query && query.ast && query.ast.clauses) || [];
 
-    this.setState({ filteredJobsSummaryList, filterClauses }, () => {
-      this.refreshSelectedJobs();
-    });
+    if (filterClauses.length === 0) {
+      this.setState({ filteredJobsSummaryList: this.state.jobsSummaryList, filterClauses }, () => {
+        this.refreshSelectedJobs();
+      });
+      return;
+    }
+
+    if (this._isFiltersSet === true) {
+      filterJobsDebounce(this.state.jobsSummaryList, filterClauses, (jobsSummaryList) => {
+        this.setState({ filteredJobsSummaryList: jobsSummaryList, filterClauses }, () => {
+          this.refreshSelectedJobs();
+        });
+      });
+    } else {
+      // first use after page load, do not debounce.
+      const filteredJobsSummaryList = filterJobs(this.state.jobsSummaryList, filterClauses);
+      this.setState({ filteredJobsSummaryList, filterClauses }, () => {
+        this.refreshSelectedJobs();
+      });
+    }
+
+    this._isFiltersSet = true;
   };
 
   onRefreshClick = () => {

x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js

@@ -347,12 +347,18 @@ export function filterJobs(jobs, clauses) {
         // if it's an array of job ids
         if (c.field === 'id') {
           js = jobs.filter((job) => c.value.indexOf(jobProperty(job, c.field)) >= 0);
-        } else {
+        } else if (c.field === 'groups') {
           // the groups value is an array of group ids
           js = jobs.filter((job) => jobProperty(job, c.field).some((g) => c.value.indexOf(g) >= 0));
+        } else if (c.field === 'job_tags') {
+          js = jobTagFilter(jobs, c.value);
         }
       } else {
-        js = jobs.filter((job) => jobProperty(job, c.field) === c.value);
+        if (c.field === 'job_tags') {
+          js = js = jobTagFilter(jobs, [c.value]);
+        } else {
+          js = jobs.filter((job) => jobProperty(job, c.field) === c.value);
+        }
       }
     }
 
@@ -369,6 +375,25 @@ export function filterJobs(jobs, clauses) {
   return filteredJobs;
 }
 
+function jobProperty(job, prop) {
+  const propMap = {
+    job_state: 'jobState',
+    datafeed_state: 'datafeedState',
+    groups: 'groups',
+    id: 'id',
+    job_tags: 'jobTags',
+  };
+  return job[propMap[prop]];
+}
+
+function jobTagFilter(jobs, value) {
+  return jobs.filter((job) => {
+    const tags = jobProperty(job, 'job_tags');
+    return Object.entries(tags)
+      .map((t) => t.join(':'))
+      .find((t) => value.some((t1) => t1 === t));
+  });
+}
 // check to see if a job has been stored in mlJobService.tempJobCloningObjects
 // if it has, return an object with the minimum properties needed for the
 // start datafeed modal.
@@ -390,13 +415,3 @@ export function checkForAutoStartDatafeed() {
     };
   }
 }
-
-function jobProperty(job, prop) {
-  const propMap = {
-    job_state: 'jobState',
-    datafeed_state: 'datafeedState',
-    groups: 'groups',
-    id: 'id',
-  };
-  return job[propMap[prop]];
-}

x-pack/plugins/ml/server/models/job_service/jobs.ts

@@ -218,6 +218,7 @@ export function jobsProvider(
         deleting: job.deleting || undefined,
         awaitingNodeAssignment: isJobAwaitingNodeAssignment(job),
         alertingRules: job.alerting_rules,
+        jobTags: job.custom_settings?.job_tags ?? {},
       };
       if (jobIds.find((j) => j === tempJob.id)) {
         tempJob.fullJob = job;

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts

@@ -55,6 +55,7 @@ describe('useInstalledSecurityJobs', () => {
             id: 'siem-api-rare_process_linux_ecs',
             isSingleMetricViewerJob: true,
             jobState: 'closed',
+            jobTags: {},
             latestTimestampMs: 1557434782207,
             memory_status: 'hard_limit',
             processed_record_count: 582251,

x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts

@@ -48,6 +48,7 @@ export const mockOpenedJob: MlSummaryJob = {
   nodeName: 'siem-es',
   processed_record_count: 3425264,
   awaitingNodeAssignment: false,
+  jobTags: {},
 };
 
 export const mockJobsSummaryResponse: MlSummaryJob[] = [
@@ -67,6 +68,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     earliestTimestampMs: 1554327458406,
     isSingleMetricViewerJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
   {
     id: 'siem-api-rare_process_linux_ecs',
@@ -83,6 +85,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     earliestTimestampMs: 1557353420495,
     isSingleMetricViewerJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
   {
     id: 'siem-api-rare_process_windows_ecs',
@@ -97,6 +100,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     datafeedState: 'stopped',
     isSingleMetricViewerJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
   {
     id: 'siem-api-suspicious_login_activity_ecs',
@@ -111,6 +115,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     datafeedState: 'stopped',
     isSingleMetricViewerJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
 ];
 
@@ -520,6 +525,7 @@ export const mockSecurityJobs: SecurityJob[] = [
     isInstalled: true,
     isElasticJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
   {
     id: 'rare_process_by_host_linux_ecs',
@@ -539,6 +545,7 @@ export const mockSecurityJobs: SecurityJob[] = [
     isInstalled: true,
     isElasticJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
   {
     datafeedId: '',
@@ -558,5 +565,6 @@ export const mockSecurityJobs: SecurityJob[] = [
     isInstalled: false,
     isElasticJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   },
 ];

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts

@@ -62,6 +62,7 @@ describe('useSecurityJobs', () => {
         isInstalled: true,
         isSingleMetricViewerJob: true,
         jobState: 'closed',
+        jobTags: {},
         latestTimestampMs: 1557434782207,
         memory_status: 'hard_limit',
         moduleId: '',

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx

@@ -44,6 +44,7 @@ describe('useSecurityJobsHelpers', () => {
         isInstalled: false,
         isSingleMetricViewerJob: false,
         jobState: '',
+        jobTags: {},
         memory_status: '',
         moduleId: 'siem_auditbeat',
         processed_record_count: 0,

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx

@@ -44,6 +44,7 @@ export const moduleToSecurityJob = (
     isInstalled: false,
     isElasticJob: true,
     awaitingNodeAssignment: false,
+    jobTags: {},
   };
 };