[Logs UI] Create ML module for log analysis (#42872) (#43261)

* Add ml module with hard-coded timestamp field

* Fix data_recognizer test

* Parameterize the bucket span normalization

* Remove max agg which will be specified during setup

The overrides are recursively merged and therefore additive. Therefore
we can't specify the timestamp agg here, because it could not be
overridden later with a different field and agg name. It needs to be
solely specified at setup time.

Author: jasonrhodes

x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js

@@ -17,6 +17,7 @@ describe('ML - data recognizer', () => {
     'apm_transaction',
     'auditbeat_process_docker_ecs',
     'auditbeat_process_hosts_ecs',
+    'logs_ui_analysis',
     'metricbeat_system_ecs',
     'nginx_ecs',
     'sample_data_ecommerce',

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "loggingApp"
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/manifest.json

@@ -0,0 +1,20 @@
+{
+  "id": "logs_ui_analysis",
+  "title": "Log Analysis",
+  "description": "Detect anomalies in log entries via the Logs UI",
+  "type": "Logs",
+  "logoFile": "logo.json",
+  "jobs": [
+    {
+      "id": "log-entry-rate",
+      "file": "log_entry_rate.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed-log-entry-rate",
+      "file": "datafeed_log_entry_rate.json",
+      "job_id": "log-entry-rate"
+    }
+  ]
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/datafeed_log_entry_rate.json

@@ -0,0 +1,28 @@
+{
+  "job_id": "JOB_ID",
+  "indexes": ["INDEX_PATTERN_NAME"],
+  "aggregations": {
+    "buckets": {
+      "date_histogram": {
+        "field": "@timestamp",
+        "fixed_interval": "900000ms"
+      },
+      "aggregations": {
+        "doc_count_per_minute": {
+          "bucket_script": {
+            "buckets_path": {
+              "doc_count": "_count"
+            },
+            "script": {
+              "lang": "painless",
+              "params": {
+                "bucket_span_in_ms": 900000
+              },
+              "source": "60 * 1000 * params.doc_count / params.bucket_span_in_ms"
+            }
+          }
+        }
+      }
+    }
+  }
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json

@@ -0,0 +1,30 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Detect anomalies in the log entry ingestion rate",
+  "groups": ["logs-ui"],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "summary_count_field_name": "doc_count_per_minute",
+    "detectors": [
+      {
+        "detector_description": "count",
+        "function": "count",
+        "detector_index": 0
+      }
+    ],
+    "influencers": []
+  },
+  "analysis_limits": {
+    "model_memory_limit": "10mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp",
+    "time_format": "epoch_ms"
+  },
+  "model_plot_config": {
+    "enabled": true
+  },
+  "custom_settings": {
+    "created_by": "ml-module-logs-ui-analysis"
+  }
+}