Skip to content

Commit 6d12c68

Browse files
Michael Hirschelasticmachine
Michael Hirsch
and
elasticmachine
authored
[ML] Adds ML modules for Metrics UI Integration (#76460)
* adds metrics ml integration * renames jobs, updates datafeeds * adds allow_no_indices: true for datafeeds * updates module ids in manifest * adds custom urls * adds module and individual job descriptions * removes model plots * updates terms agg sizes * updates chunking config * removes query and default index pattern from manifest, updates descriptions Co-authored-by: Elastic Machine <[email protected]>
1 parent 0cde5fd commit 6d12c68

17 files changed

+540
-0
lines changed

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"icon": "metricsApp"
3+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"id": "metrics_ui_hosts",
3+
"title": "Metrics Hosts",
4+
"description": "Detect anomalous memory and network behavior on hosts.",
5+
"type": "Metricbeat Data",
6+
"logoFile": "logo.json",
7+
"jobs": [
8+
{
9+
"id": "hosts_memory_usage",
10+
"file": "hosts_memory_usage.json"
11+
},
12+
{
13+
"id": "hosts_network_in",
14+
"file": "hosts_network_in.json"
15+
},
16+
{
17+
"id": "hosts_network_out",
18+
"file": "hosts_network_out.json"
19+
}
20+
],
21+
"datafeeds": [
22+
{
23+
"id": "datafeed-hosts_memory_usage",
24+
"file": "datafeed_hosts_memory_usage.json",
25+
"job_id": "hosts_memory_usage"
26+
},
27+
{
28+
"id": "datafeed-hosts_network_in",
29+
"file": "datafeed_hosts_network_in.json",
30+
"job_id": "hosts_network_in"
31+
},
32+
{
33+
"id": "datafeed-hosts_network_out",
34+
"file": "datafeed_hosts_network_out.json",
35+
"job_id": "hosts_network_out"
36+
}
37+
]
38+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"indices_options": {
7+
"allow_no_indices": true
8+
},
9+
"query": {
10+
"bool": {
11+
"must": [
12+
{"exists": {"field": "system.memory"}}
13+
]
14+
}
15+
}
16+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"indices_options": {
7+
"allow_no_indices": true
8+
},
9+
"query": {
10+
"bool": {
11+
"must": [
12+
{"exists": {"field": "system.network"}}
13+
]
14+
}
15+
},
16+
"chunking_config": {
17+
"mode": "manual",
18+
"time_span": "900s"
19+
},
20+
"aggregations": {
21+
"host.name": {"terms": {"field": "host.name", "size": 100},
22+
"aggregations": {
23+
"buckets": {
24+
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
25+
"aggregations": {
26+
"@timestamp": {"max": {"field": "@timestamp"}},
27+
"bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
28+
"bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
29+
"positive_only":{
30+
"bucket_script": {
31+
"buckets_path": {"in_derivative": "bytes_in_derivative.value"},
32+
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
33+
}
34+
}
35+
}
36+
}
37+
}
38+
}
39+
}
40+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"indices_options": {
7+
"allow_no_indices": true
8+
},
9+
"query": {
10+
"bool": {
11+
"must": [
12+
{"exists": {"field": "system.network"}}
13+
]
14+
}
15+
},
16+
"chunking_config": {
17+
"mode": "manual",
18+
"time_span": "900s"
19+
},
20+
"aggregations": {
21+
"host.name": {"terms": {"field": "host.name", "size": 100},
22+
"aggregations": {
23+
"buckets": {
24+
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
25+
"aggregations": {
26+
"@timestamp": {"max": {"field": "@timestamp"}},
27+
"bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
28+
"bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
29+
"positive_only":{
30+
"bucket_script": {
31+
"buckets_path": {"out_derivative": "bytes_out_derivative.value"},
32+
"script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
33+
}
34+
}
35+
}
36+
}
37+
}
38+
}
39+
}
40+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"job_type": "anomaly_detector",
3+
"groups": [
4+
"hosts",
5+
"metrics"
6+
],
7+
"description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
8+
"analysis_config": {
9+
"bucket_span": "15m",
10+
"detectors": [
11+
{
12+
"detector_description": "max('system.memory.actual.used.pct')",
13+
"function": "max",
14+
"field_name": "system.memory.actual.used.pct",
15+
"custom_rules": [
16+
{
17+
"actions": [
18+
"skip_result"
19+
],
20+
"conditions": [
21+
{
22+
"applies_to": "actual",
23+
"operator": "lt",
24+
"value": 0.1
25+
}
26+
]
27+
}
28+
]
29+
}
30+
],
31+
"influencers": [
32+
"host.name"
33+
]
34+
},
35+
"data_description": {
36+
"time_field": "@timestamp"
37+
},
38+
"analysis_limits": {
39+
"model_memory_limit": "64mb"
40+
},
41+
"custom_settings": {
42+
"created_by": "ml-module-metrics-ui-hosts",
43+
"custom_urls": [
44+
{
45+
"url_name": "Host Metrics",
46+
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
47+
}
48+
]
49+
}
50+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"job_type": "anomaly_detector",
3+
"description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
4+
"groups": [
5+
"hosts",
6+
"metrics"
7+
],
8+
"analysis_config": {
9+
"bucket_span": "15m",
10+
"detectors": [
11+
{
12+
"detector_description": "max(bytes_in_derivative)",
13+
"function": "max",
14+
"field_name": "bytes_in_derivative"
15+
}
16+
],
17+
"influencers": [
18+
"host.name"
19+
],
20+
"summary_count_field_name": "doc_count"
21+
},
22+
"data_description": {
23+
"time_field": "@timestamp"
24+
},
25+
"analysis_limits": {
26+
"model_memory_limit": "32mb"
27+
},
28+
"custom_settings": {
29+
"created_by": "ml-module-metrics-ui-hosts",
30+
"custom_urls": [
31+
{
32+
"url_name": "Host Metrics",
33+
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
34+
}
35+
]
36+
}
37+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"job_type": "anomaly_detector",
3+
"description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
4+
"groups": [
5+
"hosts",
6+
"metrics"
7+
],
8+
"analysis_config": {
9+
"bucket_span": "15m",
10+
"detectors": [
11+
{
12+
"detector_description": "max(bytes_out_derivative)",
13+
"function": "max",
14+
"field_name": "bytes_out_derivative"
15+
}
16+
],
17+
"influencers": [
18+
"host.name"
19+
],
20+
"summary_count_field_name": "doc_count"
21+
},
22+
"data_description": {
23+
"time_field": "@timestamp"
24+
},
25+
"analysis_limits": {
26+
"model_memory_limit": "32mb"
27+
},
28+
"custom_settings": {
29+
"created_by": "ml-module-metrics-ui-hosts",
30+
"custom_urls": [
31+
{
32+
"url_name": "Host Metrics",
33+
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
34+
}
35+
]
36+
}
37+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"icon": "metricsApp"
3+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"id": "metrics_ui_k8s",
3+
"title": "Metrics Kubernetes",
4+
"description": "Detect anomalous memory and network behavior on Kubernetes pods.",
5+
"type": "Metricbeat Data",
6+
"logoFile": "logo.json",
7+
"jobs": [
8+
{
9+
"id": "k8s_memory_usage",
10+
"file": "k8s_memory_usage.json"
11+
},
12+
{
13+
"id": "k8s_network_in",
14+
"file": "k8s_network_in.json"
15+
},
16+
{
17+
"id": "k8s_network_out",
18+
"file": "k8s_network_out.json"
19+
}
20+
],
21+
"datafeeds": [
22+
{
23+
"id": "datafeed-k8s_memory_usage",
24+
"file": "datafeed_k8s_memory_usage.json",
25+
"job_id": "k8s_memory_usage"
26+
},
27+
{
28+
"id": "datafeed-k8s_network_in",
29+
"file": "datafeed_k8s_network_in.json",
30+
"job_id": "k8s_network_in"
31+
},
32+
{
33+
"id": "datafeed-k8s_network_out",
34+
"file": "datafeed_k8s_network_out.json",
35+
"job_id": "k8s_network_out"
36+
}
37+
]
38+
}

0 commit comments

Comments
 (0)