From a1cc12472580bb49aa6da9e78bcf6e0e1d90994f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 7 Aug 2019 22:10:52 +0200 Subject: [PATCH 1/4] Add ml module with hard-coded timestamp field --- .../modules/logs_ui_analysis/logo.json | 3 ++ .../modules/logs_ui_analysis/manifest.json | 20 +++++++++++++ .../ml/datafeed_log_entry_rate.json | 27 +++++++++++++++++ .../logs_ui_analysis/ml/log_entry_rate.json | 30 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/logo.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/manifest.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/logo.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/logo.json new file mode 100644 index 0000000000000..ead765e474720 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "loggingApp" +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/manifest.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/manifest.json new file mode 100644 index 0000000000000..28fd590e68363 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/manifest.json @@ -0,0 +1,20 @@ +{ + "id": "logs_ui_analysis", + "title": "Log Analysis", + "description": "Detect anomalies in log entries via the Logs UI", + "type": "Logs", + "logoFile": "logo.json", + "jobs": [ + { + "id": "log-entry-rate", + "file": "log_entry_rate.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-log-entry-rate", + "file": "datafeed_log_entry_rate.json", + "job_id": "log-entry-rate" + } + ] +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json new file mode 100644 index 0000000000000..0188958fdead4 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json @@ -0,0 +1,27 @@ +{ + "job_id": "JOB_ID", + "indexes": ["INDEX_PATTERN_NAME"], + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "900000ms" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "doc_count_per_minute": { + "bucket_script": { + "buckets_path": { + "doc_count": "_count" + }, + "script": "params.doc_count / 15" + } + } + } + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json new file mode 100644 index 0000000000000..1e11bfa9a7f3b --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json @@ -0,0 +1,30 @@ +{ + "job_type": "anomaly_detector", + "description": "Detect anomalies in the log entry ingestion rate", + "groups": ["logs-ui"], + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count_per_minute", + "detectors": [ + { + "detector_description": "count", + "function": "count", + "detector_index": 0 + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-logs-ui-analysis" + } +} From 21eac05defdb961d0572a9d53c6ec853413dfbb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 7 Aug 2019 23:39:01 +0200 Subject: [PATCH 2/4] Fix data_recognizer test --- .../server/models/data_recognizer/__tests__/data_recognizer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js index db1822c2eec64..59bfd564f7ca2 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js @@ -17,6 +17,7 @@ describe('ML - data recognizer', () => { 'apm_transaction', 'auditbeat_process_docker_ecs', 'auditbeat_process_hosts_ecs', + 'logs_ui_analysis', 'metricbeat_system_ecs', 'nginx_ecs', 'sample_data_ecommerce', From f8e9b33d3c9c13831093102ef3b0d60044be2b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Thu, 8 Aug 2019 21:52:35 +0200 Subject: [PATCH 3/4] Parameterize the bucket span normalization --- .../logs_ui_analysis/ml/datafeed_log_entry_rate.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json index 0188958fdead4..1bc2196b3f2e1 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json @@ -18,7 +18,13 @@ "buckets_path": { "doc_count": "_count" }, - "script": "params.doc_count / 15" + "script": { + "lang": "painless", + "params": { + "bucket_span_in_ms": 900000 + }, + "source": "60 * 1000 * params.doc_count / params.bucket_span_in_ms" + } } } } From d21a0a1fd4b232edf04d31d13a301297b584c298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Fri, 9 Aug 2019 20:28:35 +0200 Subject: [PATCH 4/4] Remove max agg which will be specified during setup The overrides are recursively merged and therefore additive. Therefore we can't specify the timestamp agg here, because it could not be overridden later with a different field and agg name. It needs to be solely specified at setup time. --- .../modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json index 1bc2196b3f2e1..fbb0b6763e045 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json @@ -8,11 +8,6 @@ "fixed_interval": "900000ms" }, "aggregations": { - "@timestamp": { - "max": { - "field": "@timestamp" - } - }, "doc_count_per_minute": { "bucket_script": { "buckets_path": {