[ML] Adds ML modules for Metrics UI Integration

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json

@@ -0,0 +1,3 @@
+{
+    "icon": "metricsApp"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json

@@ -0,0 +1,59 @@
+{
+    "id": "metrics_ui_hosts",
+    "title": "Metrics Hosts",
+    "description": "Detect anomalous memory, cpu, and network behavior on hosts.",
+    "type": "Metricbeat Data",
+    "logoFile": "logo.json",
+    "defaultIndexPattern": "metricbeat-*",
+    "query": {
+        "bool": {
+          "must": [
+            {"exists": {"field": "system"}}
+          ],
+          "must_not": [
+            {"bool": {"filter": [{"exists":{"field": "kubernetes"}}]}},
+            {"bool": {"filter": [{"exists":{"field": "docker"}}]}}
+            ]
+        }
+      },
+    "jobs": [
+        {
+            "id": "hosts_cpu_usage",
+            "file": "hosts_cpu_usage.json"
+        },
+        {
+            "id": "hosts_memory_usage",
+            "file": "hosts_memory_usage.json"
+        },
+        {
+            "id": "hosts_network_in",
+            "file": "hosts_network_in.json"
+        },
+        {
+            "id": "hosts_network_out",
+            "file": "hosts_network_out.json"
+        }
+    ],
+    "datafeeds": [
+        {
+            "id": "datafeed-hosts_cpu_usage",
+            "file": "datafeed_hosts_cpu_usage.json",
+            "job_id": "hosts_cpu_usage"
+        },
+        {
+            "id": "datafeed-hosts_memory_usage",
+            "file": "datafeed_hosts_memory_usage.json",
+            "job_id": "hosts_memory_usage"
+        },
+        {
+            "id": "datafeed-hosts_network_in",
+            "file": "datafeed_hosts_network_in.json",
+            "job_id": "hosts_network_in"
+        },
+        {
+            "id": "datafeed-hosts_network_out",
+            "file": "datafeed_hosts_network_out.json",
+            "job_id": "hosts_network_out"
+        }
+    ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_cpu_usage.json

@@ -0,0 +1,69 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+      "allow_no_indices": true
+    },
+    "query": {
+      "bool": {
+        "must": [
+          {
+            "exists": {
+              "field": "system.cpu"
+            }
+          }
+        ]
+      }
+    },
+    "aggs": {
+      "host.name": {
+        "terms": {
+          "field": "host.name",
+          "size": 1000
+        },
+        "aggs": {
+          "buckets": {
+            "date_histogram": {
+              "field": "@timestamp",
+              "fixed_interval": "1m"
+            },
+            "aggs": {
+              "@timestamp": {
+                "max": {
+                  "field": "@timestamp"
+                }
+              },
+              "cpu_user": {
+                "avg": {
+                  "field": "system.cpu.user.pct"
+                }
+              },
+              "cpu_system": {
+                "avg": {
+                  "field": "system.cpu.system.pct"
+                }
+              },
+              "cpu_cores": {
+                "max": {
+                  "field": "system.cpu.cores"
+                }
+              },
+              "cpu": {
+                "bucket_script": {
+                  "buckets_path": {
+                    "user": "cpu_user",
+                    "system": "cpu_system",
+                    "cores": "cpu_cores"
+                  },
+                  "script": "(params.user + params.system) / params.cores",
+                  "gap_policy": "skip"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.memory"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json

@@ -0,0 +1,40 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.network"}}
+            ]
+        }
+    },
+    "chunking_config": {
+        "mode": "manual",
+        "time_span": "1000s"
+    },
+    "aggregations": {
+        "host.name": {"terms": {"field": "host.name"},
+            "aggregations": {
+                "buckets": {
+                    "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+                    "aggregations": {
+                        "@timestamp": {"max": {"field": "@timestamp"}},
+                        "bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
+                        "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
+                        "positive_only":{
+                            "bucket_script": {
+                                "buckets_path": {"in_derivative": "bytes_in_derivative.value"},
+                                "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json

@@ -0,0 +1,40 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+        "INDEX_PATTERN_NAME"
+    ],
+    "indices_options": {
+        "allow_no_indices": true
+    },
+    "query": {
+        "bool": {
+            "must": [
+                {"exists": {"field": "system.network"}}
+            ]
+        }
+    },
+    "chunking_config": {
+        "mode": "manual",
+        "time_span": "1000s"
+    },
+    "aggregations": {
+        "host.name": {"terms": {"field": "host.name"},
+            "aggregations": {
+                "buckets": {
+                    "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+                    "aggregations": {
+                        "@timestamp": {"max": {"field": "@timestamp"}},
+                        "bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
+                        "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
+                        "positive_only":{
+                            "bucket_script": {
+                                "buckets_path": {"out_derivative": "bytes_out_derivative.value"},
+                                "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_cpu_usage.json

@@ -0,0 +1,51 @@
+{
+    "job_type": "anomaly_detector",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "description": "Metrics: Hosts - Identify unusual spikes in cpu utilization across hosts.",
+    "analysis_config": {
+      "bucket_span": "15m",
+      "summary_count_field_name": "doc_count",
+      "detectors": [
+        {
+          "detector_description": "max('cpu')",
+          "function": "max",
+          "field_name": "cpu",
+          "custom_rules": [
+            {
+              "actions": [
+                "skip_result"
+              ],
+              "conditions": [
+                {
+                  "applies_to": "actual",
+                  "operator": "lt",
+                  "value": 0.1
+                }
+              ]
+            }
+          ]
+        }
+      ],
+      "influencers": [
+        "host.name"
+      ]
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json

@@ -0,0 +1,50 @@
+{
+    "job_type": "anomaly_detector",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+            "detector_description": "max('system.memory.actual.used.pct')",
+            "function": "max",
+            "field_name": "system.memory.actual.used.pct",
+            "custom_rules": [
+                {
+                    "actions": [
+                    "skip_result"
+                    ],
+                    "conditions": [
+                        {
+                            "applies_to": "actual",
+                            "operator": "lt",
+                            "value": 0.1
+                        }
+                    ]
+                }
+            ]
+        }
+      ],
+      "influencers": [
+        "host.name"
+      ]
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json

@@ -0,0 +1,40 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
+    "groups": [
+      "hosts",
+      "metrics"
+    ],
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+          "detector_description": "max(bytes_in_derivative)",
+          "function": "max",
+          "field_name": "bytes_in_derivative"
+        }
+      ],
+      "influencers": [
+        "host.name"
+        ],
+      "summary_count_field_name": "doc_count"
+    },
+    "model_plot_config": {
+      "enabled": "true"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "analysis_limits": {
+      "model_memory_limit": "32mb"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-metrics-ui-hosts",
+      "custom_urls": [
+        {
+          "url_name": "Host Metrics",
+          "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+        }
+      ]
+    }
+  }