From add3b53b80e2b6fb7a2f616db1082abaa538e075 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 09:32:55 -0400 Subject: [PATCH 01/25] Add CEL pipeline data collection This commit adds infrastructure to retrieve pipeline information from the Logstash Metrics API using CEL. This is intended to provide a flattened version of the data provided by the metrics API enabling visualizations to be created using standard Kibana dashboards, rather than rely on the custom stack monitoring UI --- packages/logstash/changelog.yml | 5 + .../_dev/test/system/test-default-config.yml | 6 + .../pipeline/agent/stream/stream.yml.hbs | 84 +++++++++ .../elasticsearch/ingest_pipeline/default.yml | 169 ++++++++++++++++++ .../pipeline/fields/base-fields.yml | 12 ++ .../data_stream/pipeline/fields/ecs.yml | 27 +++ .../data_stream/pipeline/fields/fields.yml | 120 +++++++++++++ .../pipeline/fields/package-fields.yml | 10 ++ .../data_stream/pipeline/manifest.yml | 20 +++ .../data_stream/pipeline/sample_event.json | 126 +++++++++++++ packages/logstash/manifest.yml | 48 ++++- 11 files changed, 625 insertions(+), 2 deletions(-) create mode 100644 packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs create mode 100644 packages/logstash/data_stream/pipeline/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/logstash/data_stream/pipeline/fields/base-fields.yml create mode 100644 packages/logstash/data_stream/pipeline/fields/ecs.yml create mode 100644 packages/logstash/data_stream/pipeline/fields/fields.yml create mode 100644 packages/logstash/data_stream/pipeline/fields/package-fields.yml create mode 100644 packages/logstash/data_stream/pipeline/manifest.yml create mode 100644 packages/logstash/data_stream/pipeline/sample_event.json diff --git a/packages/logstash/changelog.yml b/packages/logstash/changelog.yml index 804193ab395..b64991fd59d 100644 --- a/packages/logstash/changelog.yml +++ b/packages/logstash/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.4" + changes: + - description: Introduce Logstash pipelines + type: enhancement + link: - version: "2.3.3" changes: - description: Introduce Logstash node dashboards into integration package diff --git a/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..f0ab82be3a5 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +type: cel +dataset: logstash.pipeline +vars: + hosts: + - "http://{{Hostname}}:9600" +data_stream: ~ diff --git a/packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..109dec17a19 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs @@ -0,0 +1,84 @@ +config_version: "2" +interval: {{period}} +resource.url: "{{url}}/_node/stats?graph=true&vertices=true" +{{#if resource_ssl}} +resource.ssl: + {{resource_ssl}} +{{/if}} + +state: + username: {{username}} + password: {{password}} + +redact: + fields: ~ + +program: | + (has(state.username) && state.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], + } + }) + : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], + } + } + ) + : + get_request(state.url) + ) + ) + .do_request().as(resp, bytes(resp.Body) + .decode_json().as(body, + body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name} + .with({ + "elasticsearch.cluster.id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))) + }) + .with({ + "host":{ + "name":body.name, + "address":body.http_address, + } + }) + .with({ + "total":{ + "flow":body.pipelines[pipeline_name].flow, + "time":{ + "queue_push_duration":{ + "ms":body.pipelines[pipeline_name]["events"]["queue_push_duration_in_millis"], + }, + "duration":{ + "ms":body.pipelines[pipeline_name]["events"]["duration_in_millis"], + }, + }, + "reloads":{ + "successes":body.pipelines[pipeline_name]["reloads"]["successes"], + "failures":body.pipelines[pipeline_name]["reloads"]["failures"] + }, + "events":{ + "out":body.pipelines[pipeline_name]["events"]["out"], + "in":body.pipelines[pipeline_name]["events"]["in"], + "filtered":body.pipelines[pipeline_name]["events"]["filtered"], + }, + "queues":{ + "type":body.pipelines[pipeline_name]["queue"]["type"], + "events":body.pipelines[pipeline_name]["queue"]["events_count"], + "current_size":{ + "bytes":body.pipelines[pipeline_name]["queue"]["queue_size_in_bytes"], + }, + "max_size":{ + "bytes":body.pipelines[pipeline_name]["queue"]["max_queue_size_in_bytes"], + } + } + } + }) + ) + )) + .as(eve, { + "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, + "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null + }, + "events":eve.map(each, {"logstash":{"pipeline":each}})}) \ No newline at end of file diff --git a/packages/logstash/data_stream/pipeline/elasticsearch/ingest_pipeline/default.yml b/packages/logstash/data_stream/pipeline/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..8ff364355a6 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,169 @@ +--- +description: "Ingest Pipeline for Logstash Pipeline monitoring" + +processors: + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.current + if: ctx.logstash?.pipeline?.total?.flow?.queue_backpressure?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.queue_backpressure?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_backpressure.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.current + if: ctx.logstash?.pipeline?.total?.flow?.queue_persisted_growth_bytes?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.queue_persisted_growth_bytes?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.current + if: ctx.logstash?.pipeline?.total?.flow?.queue_persisted_growth_events?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.queue_persisted_growth_events?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.queue_persisted_growth_events.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.current + if: ctx.logstash?.pipeline?.total?.flow?.worker_concurrency?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.worker_concurrency?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.worker_concurrency.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.current + if: ctx.logstash?.pipeline?.total?.flow?.filter_throughput?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.filter_throughput?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.filter_throughput.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.current + if: ctx.logstash?.pipeline?.total?.flow?.output_throughput?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.output_throughput?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.output_throughput.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.current + if: ctx.logstash?.pipeline?.total?.flow?.input_throughput?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.last_1_minute + if: ctx.logstash?.pipeline?.total?.flow?.input_throughput?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.total.flow.input_throughput.last_24_hours +on_failure: + - set: + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/logstash/data_stream/pipeline/fields/base-fields.yml b/packages/logstash/data_stream/pipeline/fields/base-fields.yml new file mode 100644 index 00000000000..4227333a0d2 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: service.hostname + type: keyword + description: Hostname of the service diff --git a/packages/logstash/data_stream/pipeline/fields/ecs.yml b/packages/logstash/data_stream/pipeline/fields/ecs.yml new file mode 100644 index 00000000000..aad92fac0ef --- /dev/null +++ b/packages/logstash/data_stream/pipeline/fields/ecs.yml @@ -0,0 +1,27 @@ +- name: '@timestamp' + external: ecs +- name: service.id + external: ecs +- name: service.type + external: ecs +- name: service.version + external: ecs +- name: service.address + type: keyword + description: Address where data about this service was collected from. +- name: service.name + external: ecs +- name: process.pid + external: ecs +- name: ecs.version + external: ecs +- name: event.dataset + external: ecs +- name: event.duration + external: ecs +- name: event.module + external: ecs +- name: host.name + external: ecs +- name: error.message + external: ecs diff --git a/packages/logstash/data_stream/pipeline/fields/fields.yml b/packages/logstash/data_stream/pipeline/fields/fields.yml new file mode 100644 index 00000000000..f28d795b82e --- /dev/null +++ b/packages/logstash/data_stream/pipeline/fields/fields.yml @@ -0,0 +1,120 @@ +- name: logstash.pipeline + type: group + description: Logstash pipeline metrics + fields: + - name: name + type: keyword + description: Logstash Pipeline id/name + - name: id + type: wildcard + description: Logstash Pipeline hash + - name: elasticsearch.cluster.id + type: keyword + description: Elasticsearch clusters this Logstash pipeline is attached to + - name: host + description: Information about the host running the pipeline + type: group + fields: + - name: name + type: keyword + description: Host name of the node running logstash + - name: address + type: keyword + description: address hosting this instance of logstash + - name: total + type: group + description: Total metrics on Logstash Pipeline + fields: + - name: events.filtered + type: long + description: Number of events filtered by the pipeline + - name: events.in + type: long + description: Number of events received by the pipeline + - name: events.out + type: long + description: Number of events emitted by the pipeline + - name: time.duration.ms + unit: ms + type: long + description: Time spent processing events through the pipeline. + - name: time.queue_push_duration.ms + unit: ms + type: long + description: Time spent pushing events to the queue for this pipeline. + - name: reloads.success + type: long + description: Number of successful reloads for this pipeline + - name: reloads.failed + type: long + description: Number of failed reloads for this pipeline + - name: flow + type: group + description: flow metrics + fields: + - name: queue_persisted_growth_events.current + type: scaled_float + metric_type: gauge + description: current value of the queue persisted growth events flow metric + - name: queue_persisted_growth_bytes.current + type: scaled_float + metric_type: gauge + description: current value of the queue persisted growth bytes flow metric + - name: worker_concurrency.current + type: scaled_float + metric_type: gauge + description: current value of the worker concurrency flow metric + - name: worker_concurrency.last_1_minute + metric_type: gauge + type: scaled_float + description: current value of the worker concurrency flow metric + - name: queue_backpressure.current + type: scaled_float + metric_type: gauge + description: current value of the queue backpressure flow metric + - name: queue_backpressure.last_1_minute + metric_type: gauge + type: scaled_float + description: current value of the queue backpressure flow metric + - name: input_throughput.current + type: scaled_float + metric_type: gauge + description: current value of the input throughput flow metric + - name: input_throughput.last_1_minute + metric_type: gauge + type: scaled_float + description: current value of the throughput flow metric + - name: filter_throughput.current + type: scaled_float + metric_type: gauge + description: current value of the filter throughput flow metric + - name: filter_throughput.last_1_minute + metric_type: gauge + type: scaled_float + description: current value of the filter throughput flow metric + - name: output_throughput.current + type: scaled_float + metric_type: gauge + description: current value of the output throughput flow metric + - name: output_throughput.last_1_minute + metric_type: gauge + type: scaled_float + description: current value of the output throughput flow metric + - name: queues + type: group + fields: + - name: type + type: keyword + description: Type of queue - persistent or memory + - name: events + type: long + description: Number of events in the PQ for this pipeline + - name: current_size.bytes + type: long + metric_type: gauge + unit: byte + description: Current size of the PQ + - name: max_size.bytes + metric_type: gauge + type: long + description: Maximum possible size of the PQ diff --git a/packages/logstash/data_stream/pipeline/fields/package-fields.yml b/packages/logstash/data_stream/pipeline/fields/package-fields.yml new file mode 100644 index 00000000000..0e82e9b3a55 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/fields/package-fields.yml @@ -0,0 +1,10 @@ +- name: cluster_uuid + type: alias + path: logstash.pipeline.elasticsearch.cluster.id +- name: logstash.host.name + type: alias + path: logstash.pipeline.host.name +- name: logstash.host.address + type: alias + path: logstash.pipeline.host.address + diff --git a/packages/logstash/data_stream/pipeline/manifest.yml b/packages/logstash/data_stream/pipeline/manifest.yml new file mode 100644 index 00000000000..d13cc47d3b6 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/manifest.yml @@ -0,0 +1,20 @@ +type: metrics +title: Logstash pipeline +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: cel + enabled: false + title: "Collect Pipeline Stats" + description: "Collect Pipeline Stats" + template_path: stream.yml.hbs + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 30s diff --git a/packages/logstash/data_stream/pipeline/sample_event.json b/packages/logstash/data_stream/pipeline/sample_event.json new file mode 100644 index 00000000000..1e6ebfd2723 --- /dev/null +++ b/packages/logstash/data_stream/pipeline/sample_event.json @@ -0,0 +1,126 @@ +{ + "@timestamp": "2023-10-02T20:29:51.743Z", + "agent": { + "name": "docker-fleet-agent", + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", + "type": "filebeat", + "version": "8.10.1" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.pipeline" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "version": "8.10.1", + "snapshot": false + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-02T20:53:04Z", + "dataset": "logstash.pipeline" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + }, + "ip": [ + "192.168.86.57" + ], + "name": "docker-fleet-agent", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "82-C1-07-06-9C-04" + ], + "architecture": "x86_64" + }, + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "elasticsearch.cluster.id": [ + "t-wLhY6VT-SlT4zZ3tGLzg", + "9MOGoKiESvaklNVmxLo3iA" + ], + "total": { + "queues": { + "current_size": { + "bytes": 1 + }, + "type": "persisted", + "events": 0, + "max_size": { + "bytes": 1073741824 + } + }, + "time": { + "duration": { + "ms": 417 + }, + "queue_push_duration": { + "ms": 0 + } + }, + "flow": { + "worker_concurrency": { + "current": 0, + "last_1_minute": 0 + }, + "queue_backpressure": { + "current": 0, + "last_1_minute": 0 + }, + "input_throughput": { + "current": 0, + "last_1_minute": 0 + }, + "queue_persisted_growth_events": { + "current": 0, + "last_1_minute": 0 + }, + "queue_persisted_growth_bytes": { + "current": 0, + "last_1_minute": 0 + }, + "filter_throughput": { + "current": 0, + "last_1_minute": 0 + }, + "output_throughput": { + "current": 0, + "last_1_minute": 0 + } + }, + "events": { + "filtered": 0, + "in": 0, + "out": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + } + }, + "host": { + "address": "127.0.0.1:9600", + "name": "rb" + }, + "name": "sink" + } + } +} \ No newline at end of file diff --git a/packages/logstash/manifest.yml b/packages/logstash/manifest.yml index 375c8c6c7dc..02b197095d8 100644 --- a/packages/logstash/manifest.yml +++ b/packages/logstash/manifest.yml @@ -1,6 +1,6 @@ name: logstash title: Logstash -version: 2.3.3 +version: 2.3.4 description: Collect logs and metrics from Logstash with Elastic Agent. type: integration icons: @@ -42,7 +42,8 @@ policy_templates: show_user: false - type: logstash/metrics title: "Metrics (Stack Monitoring)" - description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana" + description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana. + Please disable if only using Technical Preview Logstash Monitoring" vars: - name: hosts type: text @@ -84,3 +85,46 @@ policy_templates: multi: false required: false show_user: false + - type: cel + title: "Metrics (Technical Preview). Please disable if using Stack Monitoring" + description: "Collect Metrics and stats from Logstash instances to power dedicated Logstash + Dashboards in Kibana" + vars: + - name: url + type: text + title: Logstash URL + show_user: true + required: true + default: http://localhost:9600 + - name: username + type: text + title: Username + description: Use when connecting to logstash + multi: false + required: false + show_user: false + - name: password + type: password + title: Password + description: Use when connecting to logstash + multi: false + required: false + show_user: false + - name: resource_ssl + type: yaml + title: SSL Configuration + description: i.e. certificate, certificate_authorities, verification_mode etc. See [SSL](https://www.elastic.co/guide/en/beats/metricbeat/current/configuration-ssl.html#ssl-client-config) for details. + multi: false + required: false + show_user: false + default: | + #certificate_authorities: ["/etc/ca.crt"] + #certificate: "/etc/client.crt" + #key: "/etc/client.key" + - name: condition + title: Condition + description: Condition to filter when to collect this input + type: text + multi: false + required: false + show_user: false From 675d77883035e1d607690b64d56ab1ff6a085439 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 09:37:51 -0400 Subject: [PATCH 02/25] Add CEL plugin data collection This commit adds infrastructure to retrieve plugin information from the Logstash Metrics API using CEL. This is intended to provide a flattened version of the data provided by the metrics API enabling visualizations to be created using standard Kibana dashboards, rather than rely on the custom stack monitoring UI. Note that by default, we are sending data at a slower cadence than for nodes and pipelines. Large Logstash pipelines may include *many* plugins, to avoid excessive data transmission, we will send at a lower cadence, and allow users to adjust depending on their needs --- .../_dev/test/system/test-default-config.yml | 6 + .../plugins/agent/stream/stream.yml.hbs | 174 +++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 123 ++++++++++++ .../plugins/fields/base-fields.yml | 12 ++ .../data_stream/plugins/fields/ecs.yml | 27 +++ .../data_stream/plugins/fields/fields.yml | 175 ++++++++++++++++++ .../plugins/fields/package-fields.yml | 9 + .../logstash/data_stream/plugins/manifest.yml | 20 ++ .../data_stream/plugins/sample_event.json | 96 ++++++++++ 9 files changed, 642 insertions(+) create mode 100644 packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs create mode 100644 packages/logstash/data_stream/plugins/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/logstash/data_stream/plugins/fields/base-fields.yml create mode 100644 packages/logstash/data_stream/plugins/fields/ecs.yml create mode 100644 packages/logstash/data_stream/plugins/fields/fields.yml create mode 100644 packages/logstash/data_stream/plugins/fields/package-fields.yml create mode 100644 packages/logstash/data_stream/plugins/manifest.yml create mode 100644 packages/logstash/data_stream/plugins/sample_event.json diff --git a/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..d2e5279d470 --- /dev/null +++ b/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +type: logstash/metrics +dataset: logstash.pipeline +vars: + hosts: + - "http://{{Hostname}}:9600" +data_stream: ~ diff --git a/packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..d2ba5271de0 --- /dev/null +++ b/packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs @@ -0,0 +1,174 @@ +config_version: "2" +interval: {{period}} +resource.url: "{{url}}/_node/stats?graph=true&vertices=true" +{{#if resource_ssl}} +resource.ssl: + {{resource_ssl}} +{{/if}} + +state: + username: {{username}} + password: {{password}} + +redact: + fields: ~ + +program: | + (has(state.username) && state.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], + } + }) + : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], + } + } + ) + : + get_request(state.url) + ) + ) + .do_request().as(resp, bytes(resp.Body) + .decode_json().as(body, + body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name}.with(body.pipelines[pipeline_name]) + .with({ + "es_cluster_id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))) + }) + .with({ + "es_cluster_id_map":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), {"plugin_id":each.id, "cluster_id":each.cluster_uuid}))) + }) + .with({ + "outputs":body.pipelines[pipeline_name]["plugins"]["outputs"] + }) + .with({ + "inputs":body.pipelines[pipeline_name]["plugins"]["inputs"] + }) + .with({ + "filters":body.pipelines[pipeline_name]["plugins"]["filters"] + }) + .with({ + "codecs":body.pipelines[pipeline_name]["plugins"]["codecs"] + }) + .with({ + "host":{ + "name":body.name, + "address":body.http_address, + } + }) + ) + )).as(events, events.map(event, + { + "inputs":event.inputs.map(input, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event['es_cluster_id'], + "plugin":{ + "type":"input", + "input":{ + "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == input.id), tuple.cluster_id), + "name":input.name, + "id":input.id, + "flow": has(input.flow) ? input["flow"] : {}, + "events":{ + "out":input.events["out"], + }, + "time":{ + "queue_push_duration":{ + "ms":input.events.queue_push_duration_in_millis + } + } + }.drop_empty() + } + }.drop_empty() + ), + "codecs":event.codecs.map(codec, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event['es_cluster_id'], + "plugin":{ + "type":"codec", + "codec":{ + "id":codec.id, + "name":codec.name, + "flow": has(codec.flow) ? codec["flow"] : {}, + "decode":{ + "duration":{ + "ms":codec.decode.duration_in_millis + }, + "in":codec.decode["writes_in"], + "out":codec.decode["out"], + }, + "encode":{ + "in":codec.encode["writes_in"], + "duration":{ + "ms":codec.encode.duration_in_millis + } + } + } + } + }.drop_empty() + ), + "filters":event.filters.map(filter, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event['es_cluster_id'], + "plugin":{ + "type":"filter", + "filter":{ + "id":filter.id, + "name":filter.name, + "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == filter.id), tuple.cluster_id), + "flow": has(filter.flow) ? filter["flow"] : {}, + "events":{ + "in":filter.events["in"], + "out":filter.events["out"], + }, + "time":{ + "duration":{ + "ms":filter.events.duration_in_millis + } + } + }.drop_empty() + } + }.drop_empty() + ), + "outputs":event.outputs.map(output, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event['es_cluster_id'], + "plugin":{ + "type":"output", + "output":{ + "id":output.id, + "name":output.name, + "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == output.id), tuple.cluster_id), + "flow": has(output.flow) ? output["flow"] : {}, + "events":{ + "in":output.events["in"], + "out":output.events["out"], + }, + "time":{ + "duration":{ + "ms":output.events.duration_in_millis + } + } + }.drop_empty() + } + }.drop_empty() + ) + }).collate(["filters", "outputs", "inputs", "codecs"])).as(eve, { + "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, + "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null + }, + "events":eve.map(each, {"logstash":{"pipeline":each}})}) \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/elasticsearch/ingest_pipeline/default.yml b/packages/logstash/data_stream/plugins/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..e11e886842b --- /dev/null +++ b/packages/logstash/data_stream/plugins/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,123 @@ +--- +description: "Pipeline for Logstash Plugin monitoring" + +processors: + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.current + if: ctx.logstash?.pipeline?.plugin?.input?.flow?.throughput?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.last_1_minute + if: ctx.logstash?.pipeline?.plugin?.input?.flow?.throughput?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.input.flow.throughput.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.current + if: ctx.logstash?.pipeline?.plugin?.filter?.flow?.worker_millis_per_event?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute + if: ctx.logstash?.pipeline?.plugin?.filter?.flow?.worker_millis_per_event?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.current + if: ctx.logstash?.pipeline?.plugin?.filter?.flow?.worker_utilization?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute + if: ctx.logstash?.pipeline?.plugin?.filter?.flow?.worker_utilization?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.filter.flow.worker_utilization.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.current + if: ctx.logstash?.pipeline?.plugin?.output?.flow?.worker_millis_per_event?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute + if: ctx.logstash?.pipeline?.plugin?.output?.flow?.worker_millis_per_event?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_24_hours + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.lifetime + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.current + if: ctx.logstash?.pipeline?.plugin?.output?.flow?.worker_utilization?.current == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute + if: ctx.logstash?.pipeline?.plugin?.output?.flow?.worker_utilization?.last_1_minute == "Infinity" + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.last_5_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.last_15_minutes + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.last_1_hour + - remove: + ignore_missing: true + field: logstash.pipeline.plugin.output.flow.worker_utilization.last_24_hours +on_failure: + - set: + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/logstash/data_stream/plugins/fields/base-fields.yml b/packages/logstash/data_stream/plugins/fields/base-fields.yml new file mode 100644 index 00000000000..4227333a0d2 --- /dev/null +++ b/packages/logstash/data_stream/plugins/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: service.hostname + type: keyword + description: Hostname of the service diff --git a/packages/logstash/data_stream/plugins/fields/ecs.yml b/packages/logstash/data_stream/plugins/fields/ecs.yml new file mode 100644 index 00000000000..aad92fac0ef --- /dev/null +++ b/packages/logstash/data_stream/plugins/fields/ecs.yml @@ -0,0 +1,27 @@ +- name: '@timestamp' + external: ecs +- name: service.id + external: ecs +- name: service.type + external: ecs +- name: service.version + external: ecs +- name: service.address + type: keyword + description: Address where data about this service was collected from. +- name: service.name + external: ecs +- name: process.pid + external: ecs +- name: ecs.version + external: ecs +- name: event.dataset + external: ecs +- name: event.duration + external: ecs +- name: event.module + external: ecs +- name: host.name + external: ecs +- name: error.message + external: ecs diff --git a/packages/logstash/data_stream/plugins/fields/fields.yml b/packages/logstash/data_stream/plugins/fields/fields.yml new file mode 100644 index 00000000000..2940dc07fc7 --- /dev/null +++ b/packages/logstash/data_stream/plugins/fields/fields.yml @@ -0,0 +1,175 @@ +- name: logstash.pipeline + type: group + description: Logstash pipeline metrics + fields: + - name: name + type: keyword + description: Logstash Pipeline id/name + - name: id + type: wildcard + description: Logstash Pipeline hash + - name: elasticsearch.cluster.id + type: keyword + description: Elasticsearch clusters this Logstash pipeline is attached to + - name: host + description: Information about the host running the pipeline + type: group + fields: + - name: name + type: keyword + description: Host name of the node running logstash + - name: address + type: keyword + description: address hosting this instance of logstash + - name: plugin + type: group + description: Information about plugins + fields: + - name: type + type: keyword + description: Type of the plugin + - name: codec + type: group + description: Information about codec plugins + fields: + - name: name + type: keyword + description: Name of codec plugin + - name: id + type: wildcard + description: Id of codec plugin + - name: encode.duration.ms + type: long + description: amount of time spend encoding events + unit: ms + - name: encode.in + type: long + description: number of events encoded + - name: decode.duration.ms + type: long + description: amount of time spend decoding events + unit: ms + - name: decode.in + type: long + description: number of events entering the decoder + - name: decode.out + type: long + description: number of events exiting the decoder + - name: input + type: group + description: Information about input plugins + fields: + - name: name + type: keyword + description: Name of input plugin + - name: id + type: wildcard + description: Id of input plugin + - name: elasticsearch.cluster.id + type: keyword + description: Elasticsearch clusters this Logstash plugin is attached to + - name: time.queue_push_duration.ms + type: long + description: amount of time spend pushing events to the queue + unit: ms + - name: events.out + type: long + description: number of events emitted by the input + - name: flow + type: group + description: flow metrics + fields: + - name: throughput.current + type: scaled_float + metric_type: gauge + description: throughput of this input plugin + - name: throughput.last_1_minute + type: scaled_float + metric_type: gauge + description: throughput of this input plugin + - name: filter + type: group + description: Information about filter plugins + fields: + - name: name + type: keyword + description: Name of filter plugin + - name: id + type: wildcard + description: Id of filter plugin + - name: elasticsearch.cluster.id + type: keyword + description: Elasticsearch clusters this Logstash plugin is attached to + - name: time.duration.ms + type: long + description: amount of time working on events in this plugin + unit: ms + - name: events.in + type: long + description: number of events received by the filter + - name: events.out + type: long + description: number of events emitted by the filter + - name: flow + type: group + description: flow metrics + fields: + - name: worker_millis_per_event.current + type: scaled_float + metric_type: gauge + description: amount of time spent per event for this plugin + - name: worker_millis_per_event.last_1_minute + type: scaled_float + metric_type: gauge + description: amount of time spent per event for this plugin + - name: worker_utilization.current + type: scaled_float + metric_type: gauge + description: worker utilization for this plugin + - name: worker_utilization.last_1_minute + type: scaled_float + metric_type: gauge + description: worker utilization for this plugin + - name: output + type: group + description: Information about filter plugins + fields: + - name: name + type: keyword + description: Name of output plugin + - name: id + type: wildcard + description: Id of output plugin + - name: elasticsearch.cluster.id + type: keyword + description: Elasticsearch clusters this Logstash plugin is attached to + - name: time.duration.ms + type: long + description: amount of time working on events in this plugin + unit: ms + - name: events.in + type: long + description: number of events received by the output + - name: events.out + type: long + description: number of events emitted by the output + - name: flow + type: group + description: flow metrics + fields: + - name: worker_millis_per_event.current + type: scaled_float + metric_type: gauge + description: amount of time spent per event for this plugin + - name: worker_millis_per_event.last_1_minute + type: scaled_float + metric_type: gauge + description: amount of time spent per event for this plugin + - name: worker_utilization.current + type: scaled_float + metric_type: gauge + description: worker utilization for this plugin + - name: worker_utilization.last_1_minute + type: scaled_float + metric_type: gauge + description: worker utilization for this plugin \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/fields/package-fields.yml b/packages/logstash/data_stream/plugins/fields/package-fields.yml new file mode 100644 index 00000000000..b0c006c06ec --- /dev/null +++ b/packages/logstash/data_stream/plugins/fields/package-fields.yml @@ -0,0 +1,9 @@ +- name: cluster_uuid + type: alias + path: logstash.pipeline.elasticsearch.cluster.id +- name: logstash.host.name + type: alias + path: logstash.pipeline.host.name +- name: logstash.host.address + type: alias + path: logstash.pipeline.host.address diff --git a/packages/logstash/data_stream/plugins/manifest.yml b/packages/logstash/data_stream/plugins/manifest.yml new file mode 100644 index 00000000000..8961e2bb9a7 --- /dev/null +++ b/packages/logstash/data_stream/plugins/manifest.yml @@ -0,0 +1,20 @@ +type: metrics +title: Logstash pipeline +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: cel + enabled: false + title: "Collect Plugin Stats" + description: "Collect Plugin Stats" + template_path: stream.yml.hbs + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 1m \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/sample_event.json b/packages/logstash/data_stream/plugins/sample_event.json new file mode 100644 index 00000000000..9a9dbf11c3a --- /dev/null +++ b/packages/logstash/data_stream/plugins/sample_event.json @@ -0,0 +1,96 @@ +{ + "@timestamp": "2023-10-02T20:29:51.743Z", + "agent": { + "name": "docker-fleet-agent", + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", + "type": "filebeat", + "version": "8.10.1" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.plugins" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "version": "8.10.1", + "snapshot": false + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-02T20:50:52Z", + "dataset": "logstash.plugins" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + }, + "ip": [ + "192.168.86.57" + ], + "name": "docker-fleet-agent", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "82-C1-07-06-9C-04" + ], + "architecture": "x86_64" + }, + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "elasticsearch.cluster.id": [ + "9MOGoKiESvaklNVmxLo3iA" + ], + "plugin": { + "output": { + "elasticsearch.cluster.id": [ + "9MOGoKiESvaklNVmxLo3iA" + ], + "name": "elasticsearch", + "time": { + "duration": { + "ms": 90332 + } + }, + "id": "09a1c3091b4b30005c1d9f295fefab6072135a7064d8ccfb1171511e91c99c13", + "events": { + "in": 10252, + "out": 10252 + }, + "flow": { + "worker_millis_per_event": { + "current": 6.091, + "last_1_minute": 7.545 + }, + "worker_utilization": { + "current": 0.02805, + "last_1_minute": 0.01597 + } + } + }, + "type": "output" + }, + "name": "conversion_with_queues", + "host": { + "address": "127.0.0.1:9602", + "name": "rb2" + }, + "id": "75cb8b8394ad9d5eacdaa58768b827ba74103f41dcbf8d261e57cc2f9087261e" + } + } +} \ No newline at end of file From a13ff6ba2b85edb5894e96598d17ccfc2f3b1105 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 09:51:03 -0400 Subject: [PATCH 03/25] Add CEL node data collection This commit adds infrastructure to retrieve node information from the Logstash Metrics API using CEL. This is intended to replace the existing node retrieval for stack monitoring and is intended to mirror the data that was previously retrieved by metricbeat --- .../_dev/test/system/test-default-config.yml | 6 + .../node_cel/agent/stream/stream.yml.hbs | 68 +++++ .../node_cel/fields/base-fields.yml | 12 + .../data_stream/node_cel/fields/ecs.yml | 27 ++ .../data_stream/node_cel/fields/fields.yml | 232 ++++++++++++++++++ .../node_cel/fields/package-fields.yml | 12 + .../data_stream/node_cel/manifest.yml | 21 ++ .../data_stream/node_cel/sample_event.json | 175 +++++++++++++ 8 files changed, 553 insertions(+) create mode 100644 packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml create mode 100644 packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs create mode 100644 packages/logstash/data_stream/node_cel/fields/base-fields.yml create mode 100644 packages/logstash/data_stream/node_cel/fields/ecs.yml create mode 100644 packages/logstash/data_stream/node_cel/fields/fields.yml create mode 100644 packages/logstash/data_stream/node_cel/fields/package-fields.yml create mode 100644 packages/logstash/data_stream/node_cel/manifest.yml create mode 100644 packages/logstash/data_stream/node_cel/sample_event.json diff --git a/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..51f3a692227 --- /dev/null +++ b/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +type: logstash/metrics +dataset: logstash.node +vars: + hosts: + - "http://{{Hostname}}:9600" +data_stream: ~ diff --git a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..2f73ba745fb --- /dev/null +++ b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs @@ -0,0 +1,68 @@ +config_version: "2" +interval: {{period}} +resource.url: "{{url}}/_node/stats?graph=true" +{{#if resource_ssl}} +resource.ssl: + {{resource_ssl}} +{{/if}} + +state: + username: {{username}} + password: {{password}} + +redact: + fields: ~ + +program: | + (has(state.username) && state.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], + } + }) + : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? + get_request(state.url).with({ + "Header":{ + "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], + } + } + ) + : + get_request(state.url) + ) + ) + .do_request().as(resp, bytes(resp.Body) + .decode_json() + .as(body, {"b":body} + .with({"logstash":{"node":{"stats":{ + "events":body['events'], + "jvm":body['jvm'], + "queue":body['queue'], + "reloads":body['reloads'], + "process":body['process'], + "os":{ + "cpu":body['process']['cpu'], + "cgroup":has(body.os.group) ? body['os']['cgroup'] : {}, + }, + "logstash":{ + "ephemeral_id":body['ephemeral_id'], + "host":body['host'], + "http_address":body['http_address'], + "name":body['name'], + "pipeline":body['pipeline'], + "pipelines":body['pipelines'].map(pipeline, [pipeline]).flatten(), + "snapshot":body['snapshot'], + "status":body['status'], + "uuid":body['id'], + "version":body['version'], + } + }} + }}) + ) + ).drop("b") + .as(eve, { + "events":eve.map(each, eve), + "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, + "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null + }, + }) \ No newline at end of file diff --git a/packages/logstash/data_stream/node_cel/fields/base-fields.yml b/packages/logstash/data_stream/node_cel/fields/base-fields.yml new file mode 100644 index 00000000000..4227333a0d2 --- /dev/null +++ b/packages/logstash/data_stream/node_cel/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: service.hostname + type: keyword + description: Hostname of the service diff --git a/packages/logstash/data_stream/node_cel/fields/ecs.yml b/packages/logstash/data_stream/node_cel/fields/ecs.yml new file mode 100644 index 00000000000..aad92fac0ef --- /dev/null +++ b/packages/logstash/data_stream/node_cel/fields/ecs.yml @@ -0,0 +1,27 @@ +- name: '@timestamp' + external: ecs +- name: service.id + external: ecs +- name: service.type + external: ecs +- name: service.version + external: ecs +- name: service.address + type: keyword + description: Address where data about this service was collected from. +- name: service.name + external: ecs +- name: process.pid + external: ecs +- name: ecs.version + external: ecs +- name: event.dataset + external: ecs +- name: event.duration + external: ecs +- name: event.module + external: ecs +- name: host.name + external: ecs +- name: error.message + external: ecs diff --git a/packages/logstash/data_stream/node_cel/fields/fields.yml b/packages/logstash/data_stream/node_cel/fields/fields.yml new file mode 100644 index 00000000000..c0cf437ff00 --- /dev/null +++ b/packages/logstash/data_stream/node_cel/fields/fields.yml @@ -0,0 +1,232 @@ +- name: logstash + type: group + fields: + - name: elasticsearch.cluster.id + type: keyword + - name: node + type: group + fields: + - name: state.pipeline + type: group + fields: + - name: id + type: keyword + - name: hash + type: keyword + - name: stats + type: group + fields: + - name: reloads + type: group + fields: + - name: failures + type: long + - name: successes + type: long + - name: timestamp + type: date + - name: jvm + type: group + fields: + - name: threads + type: group + fields: + - name: count + type: long + description: current number of threads + - name: peak_count + type: long + description: peak number of threads + - name: uptime_in_millis + type: long + - name: mem + type: group + fields: + - name: heap_used_in_bytes + type: long + - name: heap_max_in_bytes + type: long + - name: heap_used_percent + type: long + - name: gc + type: group + fields: + - name: collectors + type: group + fields: + - name: old + type: group + fields: + - name: collection_count + type: long + - name: collection_time_in_millis + type: long + - name: young + type: group + fields: + - name: collection_count + type: long + - name: collection_time_in_millis + type: long + - name: events + type: group + fields: + - name: in + type: long + description: | + Incoming events counter. + - name: out + type: long + description: | + Outgoing events counter. + - name: filtered + type: long + description: | + Filtered events counter. + - name: duration_in_millis + type: long + - name: logstash + type: group + fields: + - name: uuid + type: keyword + - name: version + type: keyword + - name: ephemeral_id + type: keyword + - name: host + type: keyword + - name: http_address + type: keyword + - name: name + type: keyword + - name: snapshot + type: boolean + - name: status + type: keyword + - name: pipelines + type: keyword + - name: pipeline + type: group + fields: + - name: batch_size + type: long + - name: workers + type: long + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: percent + type: double + - name: load_average + type: group + fields: + - name: 15m + type: half_float + - name: 1m + type: half_float + - name: 5m + type: half_float + - name: cgroup + type: group + fields: + - name: cpuacct + type: group + fields: + - name: control_group + type: text + - name: usage_nanos + type: long + - name: cpu + type: group + fields: + - name: control_group + type: text + - name: cfs_quota_micros + type: long + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: long + - name: time_throttled_nanos + type: long + - name: number_of_times_throttled + type: long + - name: process + type: group + fields: + - name: cpu.percent + type: double + - name: max_file_descriptors + type: long + - name: open_file_descriptors + type: long + - name: queue.events_count + type: long + - name: pipelines + type: nested + - name: pipelines.id + type: keyword + - name: pipelines.hash + type: keyword + - name: pipelines.ephemeral_id + type: keyword + - name: pipelines.reloads + type: group + fields: + - name: failures + type: long + - name: successes + type: long + - name: pipelines.queue + type: group + fields: + - name: events_count + type: long + - name: type + type: keyword + - name: queue_size_in_bytes + type: long + - name: max_queue_size_in_bytes + type: long + - name: pipelines.events + type: group + fields: + - name: in + type: long + - name: out + type: long + - name: filtered + type: long + - name: duration_in_millis + type: long + - name: queue_push_duration_in_millis + type: long + - name: pipelines.vertices + type: nested + - name: pipelines.vertices.id + type: keyword + description: id + - name: pipelines.vertices.long_counters + type: nested + - name: pipelines.vertices.long_counters.name + type: keyword + - name: pipelines.vertices.long_counters.value + type: long + - name: pipelines.vertices.duration_in_millis + type: long + - name: pipelines.vertices.events_in + type: long + - name: pipelines.vertices.pipeline_ephemeral_id + type: keyword + description: pipeline_ephemeral_id + - name: pipelines.vertices.events_out + type: long + description: events_out + - name: pipelines.vertices.queue_push_duration_in_millis + type: long + description: queue_push_duration_in_millis diff --git a/packages/logstash/data_stream/node_cel/fields/package-fields.yml b/packages/logstash/data_stream/node_cel/fields/package-fields.yml new file mode 100644 index 00000000000..7464f183f19 --- /dev/null +++ b/packages/logstash/data_stream/node_cel/fields/package-fields.yml @@ -0,0 +1,12 @@ +- name: cluster_uuid + type: alias + path: logstash.elasticsearch.cluster.id +- name: logstash.host.name + type: alias + path: logstash.node.stats.logstash.name +- name: logstash.host.address + type: alias + path: logstash.node.stats.logstash.host.address +- name: logstash.pipeline.name + type: alias + path: logstash.node.stats.logstash.pipelines \ No newline at end of file diff --git a/packages/logstash/data_stream/node_cel/manifest.yml b/packages/logstash/data_stream/node_cel/manifest.yml new file mode 100644 index 00000000000..71da9eee957 --- /dev/null +++ b/packages/logstash/data_stream/node_cel/manifest.yml @@ -0,0 +1,21 @@ +type: metrics +title: Logstash Node Stats +dataset: logstash.node +elasticsearch: + index_template: + mappings: + dynamic: true +streams: + - input: cel + enabled: false + title: "Collect Node Stats" + description: "Collect Node Stats." + template_path: stream.yml.hbs + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 30s \ No newline at end of file diff --git a/packages/logstash/data_stream/node_cel/sample_event.json b/packages/logstash/data_stream/node_cel/sample_event.json new file mode 100644 index 00000000000..eb3ed4c095b --- /dev/null +++ b/packages/logstash/data_stream/node_cel/sample_event.json @@ -0,0 +1,175 @@ +{ + "@timestamp": "2023-10-02T20:29:51.743Z", + "agent": { + "name": "docker-fleet-agent", + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", + "type": "filebeat", + "version": "8.10.1" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.node" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "version": "8.10.1", + "snapshot": false + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-02T20:29:52Z", + "dataset": "logstash.node" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + }, + "ip": [ + "192.168.86.57" + ], + "name": "docker-fleet-agent", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "82-C1-07-06-9C-04" + ], + "architecture": "x86_64" + }, + "input": { + "type": "cel" + }, + "logstash": { + "node": { + "stats": { + "jvm": { + "mem": { + "heap_committed_in_bytes": 1073741824, + "heap_used_percent": 13, + "heap_max_in_bytes": 1073741822, + "non_heap_committed_in_bytes": 200540160, + "pools": { + "young": { + "used_in_bytes": 3145728, + "peak_used_in_bytes": 629145600, + "committed_in_bytes": 649068544, + "max_in_bytes": -1, + "peak_max_in_bytes": -1 + }, + "old": { + "used_in_bytes": 111104000, + "peak_used_in_bytes": 111104000, + "committed_in_bytes": 397410304, + "max_in_bytes": 1073741824, + "peak_max_in_bytes": 1073741824 + }, + "survivor": { + "used_in_bytes": 26945264, + "peak_used_in_bytes": 39405808, + "committed_in_bytes": 27262976, + "max_in_bytes": -1, + "peak_max_in_bytes": -1 + } + }, + "non_heap_used_in_bytes": 180942928, + "heap_used_in_bytes": 141194992 + }, + "threads": { + "count": 46, + "peak_count": 50 + }, + "uptime_in_millis": 26669940, + "gc": { + "collectors": { + "young": { + "collection_count": 23, + "collection_time_in_millis": 341 + }, + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + } + } + } + }, + "logstash": { + "pipeline": { + "batch_delay": 50, + "batch_size": 125, + "workers": 16 + }, + "pipelines": [ + "sink", + "source" + ], + "http_address": "127.0.0.1:9600", + "name": "rb", + "host": "robbaveys-MacBook-Pro.local", + "ephemeral_id": "43a7fd83-47ce-4646-bbb1-e93391041ada", + "version": "8.8.2", + "uuid": "b924ad43-2dbe-42b4-8931-75a52aa7fcbf", + "snapshot": false, + "status": "green" + }, + "process": { + "open_file_descriptors": 104, + "mem": { + "total_virtual_in_bytes": 40497684480 + }, + "max_file_descriptors": 10240, + "cpu": { + "load_average": { + "1m": 7.01220703125 + }, + "total_in_millis": 446183, + "percent": 0 + }, + "peak_open_file_descriptors": 106 + }, + "os": { + "dpu": { + "load_average": { + "1m": 7.01220703125 + }, + "percent": 0 + }, + "cpu": { + "load_average": { + "1m": 7.01220703125 + }, + "total_in_millis": 446183, + "percent": 0 + }, + "cgroup": {} + }, + "queue": { + "events_count": 0 + }, + "events": { + "filtered": 0, + "in": 0, + "queue_push_duration_in_millis": 0, + "duration_in_millis": 1108, + "out": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + } + } + } + } + +} \ No newline at end of file From f2584d1252d5ec595bbebda4ce63a15f6f664171 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 10:14:13 -0400 Subject: [PATCH 04/25] Add dashboards for pipelines and plugins This commit adds dashboards for Logstash pipelines and plugins --- packages/logstash/changelog.yml | 4 +- ...-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 6762 +++++++++++++++++ ...-c0594170-526a-11ee-9ecc-31444cb79548.json | 1612 ++++ 3 files changed, 8376 insertions(+), 2 deletions(-) create mode 100644 packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json create mode 100644 packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json diff --git a/packages/logstash/changelog.yml b/packages/logstash/changelog.yml index b64991fd59d..10bfa6054f9 100644 --- a/packages/logstash/changelog.yml +++ b/packages/logstash/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top - version: "2.3.4" changes: - - description: Introduce Logstash pipelines + - description: Introduce Logstash pipelines plugin retrieval and dashboards type: enhancement - link: + link: https://github.com/elastic/integrations/pull/7704 - version: "2.3.3" changes: - description: Introduce Logstash node dashboards into integration package diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json new file mode 100644 index 00000000000..3794733ff40 --- /dev/null +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -0,0 +1,6762 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"429cdb8b-f9d9-45b0-bf04-2fca76ad3cd8\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"429cdb8b-f9d9-45b0-bf04-2fca76ad3cd8\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"selectedOptions\":[],\"enhancements\":{},\"existsSelected\":false}},\"497f4765-376b-4a02-9982-0e8bf9a0b1ef\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"497f4765-376b-4a02-9982-0e8bf9a0b1ef\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{},\"selectedOptions\":[],\"existsSelected\":false}}}" + }, + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipeline Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n**[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)** \n\n\nOverview\n\nThis Dashboard gives a detailed view of a Logstash Pipeline. Should be used in conjunction with pipeline and node controls", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 171, + "i": "1b7d4c91-b582-4639-a771-7853e72a94b6", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "1b7d4c91-b582-4639-a771-7853e72a94b6", + "title": "Logstash Nav Panel", + "type": "visualization", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-6cc6ec80-cf6c-4421-a323-256f08b59426", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "6cc6ec80-cf6c-4421-a323-256f08b59426": { + "columnOrder": [ + "4c006c30-3352-4dbf-bd1c-317b3ceb9377", + "e0ba2202-b4d6-437f-a0c0-84597e7ee00e" + ], + "columns": { + "4c006c30-3352-4dbf-bd1c-317b3ceb9377": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "e0ba2202-b4d6-437f-a0c0-84597e7ee00e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Logstash instances running pipeline", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.host.name" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "linkToLayers": [ + "d6fe4fea-e92e-4042-9c92-0663b598a523" + ], + "sampling": 1 + }, + "d6fe4fea-e92e-4042-9c92-0663b598a523": { + "columnOrder": [ + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52" + ], + "columns": { + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Logstash instances running pipeline", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.host.name" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "d6fe4fea-e92e-4042-9c92-0663b598a523", + "layerType": "data", + "metricAccessor": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "showBar": false, + "trendlineLayerId": "6cc6ec80-cf6c-4421-a323-256f08b59426", + "trendlineLayerType": "metricTrendline", + "trendlineMetricAccessor": "e0ba2202-b4d6-437f-a0c0-84597e7ee00e", + "trendlineTimeAccessor": "4c006c30-3352-4dbf-bd1c-317b3ceb9377" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "9a756479-5952-441e-beff-e0896500ca39", + "w": 14, + "x": 8, + "y": 0 + }, + "panelIndex": "9a756479-5952-441e-beff-e0896500ca39", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "d6fe4fea-e92e-4042-9c92-0663b598a523": { + "columnOrder": [ + "c7618e15-45c6-480c-8035-d6a741940c97", + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52" + ], + "columns": { + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.out\": *" + }, + "isBucketed": false, + "label": "Events emitted from pipeline", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "c7618e15-45c6-480c-8035-d6a741940c97": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of logstash.host.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "c7618e15-45c6-480c-8035-d6a741940c97", + "collapseFn": "sum", + "layerId": "d6fe4fea-e92e-4042-9c92-0663b598a523", + "layerType": "data", + "metricAccessor": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "showBar": false + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "6da9466a-ede0-44b6-a2e7-d704065bb90b", + "w": 13, + "x": 35, + "y": 0 + }, + "panelIndex": "6da9466a-ede0-44b6-a2e7-d704065bb90b", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "a387e7a5-3021-4123-8be6-dc78275ee3cb": { + "columnOrder": [ + "6e436e10-2a41-4031-9b77-43f68f433d91", + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf", + "089e8930-be0f-4b64-8b26-4c3bf129b40e", + "c95ba72a-fc3f-4918-8d21-49ae10486fcb" + ], + "columns": { + "089e8930-be0f-4b64-8b26-4c3bf129b40e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Received", + "operationType": "counter_rate", + "references": [ + "c95ba72a-fc3f-4918-8d21-49ae10486fcb" + ], + "scale": "ratio", + "timeScale": "s" + }, + "6e436e10-2a41-4031-9b77-43f68f433d91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Hosts", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.out\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.total.events.out", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "c95ba72a-fc3f-4918-8d21-49ae10486fcb": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.events.in", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "089e8930-be0f-4b64-8b26-4c3bf129b40e" + ], + "layerId": "a387e7a5-3021-4123-8be6-dc78275ee3cb", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "6e436e10-2a41-4031-9b77-43f68f433d91", + "xAccessor": "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "f03395de-a163-4204-ac82-83c0ea542eeb", + "w": 20, + "x": 8, + "y": 5 + }, + "panelIndex": "f03395de-a163-4204-ac82-83c0ea542eeb", + "title": "Events Received Rate", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "d6fe4fea-e92e-4042-9c92-0663b598a523": { + "columnOrder": [ + "27acc57b-d205-4010-a882-9ea93f8fc3ab", + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52" + ], + "columns": { + "27acc57b-d205-4010-a882-9ea93f8fc3ab": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of logstash.host.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.in\": *" + }, + "isBucketed": false, + "label": "Events received by pipeline", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "27acc57b-d205-4010-a882-9ea93f8fc3ab", + "collapseFn": "sum", + "layerId": "d6fe4fea-e92e-4042-9c92-0663b598a523", + "layerType": "data", + "metricAccessor": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "showBar": false + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", + "w": 13, + "x": 22, + "y": 0 + }, + "panelIndex": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "a387e7a5-3021-4123-8be6-dc78275ee3cb": { + "columnOrder": [ + "6e436e10-2a41-4031-9b77-43f68f433d91", + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf", + "51e34028-4e94-4704-9ecf-1c1b15dbf27a", + "70e0b76e-1e5a-4abf-9e17-fc26887f2e5a" + ], + "columns": { + "51e34028-4e94-4704-9ecf-1c1b15dbf27a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Emitted", + "operationType": "counter_rate", + "references": [ + "70e0b76e-1e5a-4abf-9e17-fc26887f2e5a" + ], + "scale": "ratio", + "timeScale": "s" + }, + "6e436e10-2a41-4031-9b77-43f68f433d91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Hosts", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.out\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.total.events.out", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "70e0b76e-1e5a-4abf-9e17-fc26887f2e5a": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.events.out", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27a" + ], + "layerId": "a387e7a5-3021-4123-8be6-dc78275ee3cb", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "6e436e10-2a41-4031-9b77-43f68f433d91", + "xAccessor": "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "4fad1f03-99d3-4834-a6c4-bea283178567", + "w": 20, + "x": 28, + "y": 5 + }, + "panelIndex": "4fad1f03-99d3-4834-a6c4-bea283178567", + "title": "Events Emitted Rate", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "a387e7a5-3021-4123-8be6-dc78275ee3cb": { + "columnOrder": [ + "6e436e10-2a41-4031-9b77-43f68f433d91", + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf", + "51e34028-4e94-4704-9ecf-1c1b15dbf27a" + ], + "columns": { + "51e34028-4e94-4704-9ecf-1c1b15dbf27a": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.queues.events\": *" + }, + "isBucketed": false, + "label": "Persistent Queue Size (number of Events)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0 + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.events" + }, + "6e436e10-2a41-4031-9b77-43f68f433d91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Hosts", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.out\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.total.events.out", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "emphasizeFitting": true, + "endValue": "Nearest", + "fittingFunction": "Carry", + "layers": [ + { + "accessors": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27a" + ], + "layerId": "a387e7a5-3021-4123-8be6-dc78275ee3cb", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "6e436e10-2a41-4031-9b77-43f68f433d91", + "xAccessor": "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "a3444242-6e3e-4959-8815-43e65b3e8d69", + "w": 20, + "x": 8, + "y": 14 + }, + "panelIndex": "a3444242-6e3e-4959-8815-43e65b3e8d69", + "title": "Persistent Queue Size (events)", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "a387e7a5-3021-4123-8be6-dc78275ee3cb": { + "columnOrder": [ + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf", + "6e436e10-2a41-4031-9b77-43f68f433d91", + "51e34028-4e94-4704-9ecf-1c1b15dbf27a", + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX0", + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX1", + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX2" + ], + "columns": { + "51e34028-4e94-4704-9ecf-1c1b15dbf27a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Persistent Queue Utilization(%)", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "round((last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes)), 2)", + "isFormulaBroken": false + }, + "references": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX2" + ], + "scale": "ratio" + }, + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.queues.current_size.bytes\": *" + }, + "isBucketed": false, + "label": "Part of Persistent Queue Size", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.current_size.bytes" + }, + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.queues.max_size.bytes\": *" + }, + "isBucketed": false, + "label": "Part of Persistent Queue Size", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.max_size.bytes" + }, + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Persistent Queue Size", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + { + "args": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX0", + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX1" + ], + "location": { + "max": 127, + "min": 6 + }, + "name": "divide", + "text": "(last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes))", + "type": "function" + }, + 2 + ], + "location": { + "max": 131, + "min": 0 + }, + "name": "round", + "text": "round((last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes)), 2)", + "type": "function" + } + }, + "references": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX0", + "51e34028-4e94-4704-9ecf-1c1b15dbf27aX1" + ], + "scale": "ratio" + }, + "6e436e10-2a41-4031-9b77-43f68f433d91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Hosts", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.out\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.total.events.out", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "emphasizeFitting": true, + "endValue": "Nearest", + "fittingFunction": "Carry", + "layers": [ + { + "accessors": [ + "51e34028-4e94-4704-9ecf-1c1b15dbf27a" + ], + "layerId": "a387e7a5-3021-4123-8be6-dc78275ee3cb", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "6e436e10-2a41-4031-9b77-43f68f433d91", + "xAccessor": "9e3d2cfb-6eea-4a9b-aac2-07020677a1cf" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide", + "yLeftExtent": { + "lowerBound": 0, + "mode": "custom", + "upperBound": 1 + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "5171ef08-d301-4a9c-832b-40f41c9ad24d", + "w": 20, + "x": 28, + "y": 14 + }, + "panelIndex": "5171ef08-d301-4a9c-832b-40f41c9ad24d", + "title": "Persistent Queue Utilization (%)", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "Inputs", + "openLinksInNewTab": false + }, + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "681d5f97-e2f4-4a74-9710-7e75bb73fea8", + "w": 24, + "x": 16, + "y": 23 + }, + "panelIndex": "681d5f97-e2f4-4a74-9710-7e75bb73fea8", + "title": "Plugins", + "type": "visualization", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "df79554f-08e3-4f7f-b438-80bc81e14637": { + "columnOrder": [ + "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "245e3126-537a-4644-b6b3-9a33800dfb91", + "0a5be802-a381-4edd-ad8c-488a6a419619", + "410757d6-0433-4cdd-b51b-138704dfc00a", + "d172976a-5c43-46b7-9d96-44ac40134cbe", + "3bdc4103-5313-4e00-a407-de5b7fde820c", + "a175cdde-215b-441e-84d6-ceab03b1495d", + "d172976a-5c43-46b7-9d96-44ac40134cbeX0", + "d172976a-5c43-46b7-9d96-44ac40134cbeX1", + "3bdc4103-5313-4e00-a407-de5b7fde820cX0", + "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + ], + "columns": { + "0a5be802-a381-4edd-ad8c-488a6a419619": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Id", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.input.id" + }, + "245e3126-537a-4644-b6b3-9a33800dfb91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time Spent Pushing to Queue", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + ], + "scale": "ratio" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.time.queue_push_duration.ms" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "operationType": "counter_rate", + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "410757d6-0433-4cdd-b51b-138704dfc00a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "65ca78d2-d895-4db6-bec3-7e110bd0030b": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.input.name" + }, + "a175cdde-215b-441e-84d6-ceab03b1495d": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.input.flow.throughput.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Throughput Flow", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.flow.throughput.last_1_minute" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbe": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Emitted", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.input.events.out))", + "isFormulaBroken": false + }, + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX1" + ], + "scale": "ratio" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.events.out))", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.events.out" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.events.out))", + "operationType": "counter_rate", + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX0" + ], + "scale": "ratio", + "timeScale": "s" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "collapseFn": "sum", + "columnId": "410757d6-0433-4cdd-b51b-138704dfc00a", + "hidden": true, + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "0a5be802-a381-4edd-ad8c-488a6a419619", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "d172976a-5c43-46b7-9d96-44ac40134cbe", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "3bdc4103-5313-4e00-a407-de5b7fde820c", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "columnId": "245e3126-537a-4644-b6b3-9a33800dfb91", + "isTransposed": false + } + ], + "layerId": "df79554f-08e3-4f7f-b438-80bc81e14637", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 7, + "i": "9a34af7e-5f12-49b4-86e0-5577203711eb", + "w": 40, + "x": 8, + "y": 27 + }, + "panelIndex": "9a34af7e-5f12-49b4-86e0-5577203711eb", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.input.name", + "logstash.pipeline.plugin.input.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time Spent Pushing to Queue", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.time.queue_push_duration.ms" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "f94ee279-63d8-45de-bfa0-9c93416b9752", + "w": 40, + "x": 8, + "y": 34 + }, + "panelIndex": "f94ee279-63d8-45de-bfa0-9c93416b9752", + "title": "Time Spent Pushing to Queues", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.input.name", + "logstash.pipeline.plugin.input.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events per second", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0 + } + }, + "formula": "counter_rate(last_value(logstash.pipeline.plugin.input.events.out))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.input.events.out\": *" + }, + "isBucketed": false, + "label": "Part of counter_rate(last_value(logstash.pipeline.plugin.input.events.out))", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.events.out" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of counter_rate(last_value(logstash.pipeline.plugin.input.events.out))", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "1eabaf75-95dd-4cae-95d5-f00efe978872", + "w": 20, + "x": 8, + "y": 43 + }, + "panelIndex": "1eabaf75-95dd-4cae-95d5-f00efe978872", + "title": "Events Emitted per second", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.input.name", + "logstash.pipeline.plugin.input.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.input.flow.throughput.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Flow Throughput", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "compact": true, + "decimals": 4 + } + }, + "showArrayValues": false, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.flow.throughput.last_1_minute" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 9, + "i": "3470dd76-7ebd-4b19-a3de-fa0082a72ee5", + "w": 20, + "x": 28, + "y": 43 + }, + "panelIndex": "3470dd76-7ebd-4b19-a3de-fa0082a72ee5", + "title": "Input Flow", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "Filters", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "7f5bcb0f-394c-4bbd-b17a-c163a28a9315", + "w": 24, + "x": 16, + "y": 52 + }, + "panelIndex": "7f5bcb0f-394c-4bbd-b17a-c163a28a9315", + "title": "Plugins", + "type": "visualization", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "df79554f-08e3-4f7f-b438-80bc81e14637": { + "columnOrder": [ + "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "18e039ef-9926-4559-bb1a-d2894c21f750", + "0a5be802-a381-4edd-ad8c-488a6a419619", + "410757d6-0433-4cdd-b51b-138704dfc00a", + "8914109f-497d-4293-8692-dbdddffbf42f", + "d172976a-5c43-46b7-9d96-44ac40134cbe", + "56e40393-65fa-403e-b757-05ae1fc04063", + "3bdc4103-5313-4e00-a407-de5b7fde820c", + "5f2a9ea0-38a8-403a-bf56-5792745b0b7a", + "6fdcaadf-6c83-43f1-9f65-b148037b7aab", + "d172976a-5c43-46b7-9d96-44ac40134cbeX0", + "d172976a-5c43-46b7-9d96-44ac40134cbeX1", + "3bdc4103-5313-4e00-a407-de5b7fde820cX0", + "3bdc4103-5313-4e00-a407-de5b7fde820cX1", + "b1988bb4-4923-4f92-b63e-1cc9d230457f", + "56e40393-65fa-403e-b757-05ae1fc04063X0", + "56e40393-65fa-403e-b757-05ae1fc04063X1", + "56e40393-65fa-403e-b757-05ae1fc04063X2", + "56e40393-65fa-403e-b757-05ae1fc04063X3", + "56e40393-65fa-403e-b757-05ae1fc04063X4", + "56e40393-65fa-403e-b757-05ae1fc04063X5", + "56e40393-65fa-403e-b757-05ae1fc04063X6" + ], + "columns": { + "0a5be802-a381-4edd-ad8c-488a6a419619": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Id", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.events.out: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.filter.id" + }, + "18e039ef-9926-4559-bb1a-d2894c21f750": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "5f2a9ea0-38a8-403a-bf56-5792745b0b7a", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent in plugin", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.filter.time.duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + ], + "scale": "ratio" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent in plugin", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent in plugin", + "operationType": "counter_rate", + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "410757d6-0433-4cdd-b51b-138704dfc00a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "56e40393-65fa-403e-b757-05ae1fc04063": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events processed/ms", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "/ms" + } + }, + "formula": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.time.duration.ms)))", + "isFormulaBroken": false + }, + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X6" + ], + "scale": "ratio" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.events.in\": *" + }, + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.in" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "counter_rate", + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "cumulative_sum", + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X1" + ], + "scale": "ratio" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X3": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.time.duration.ms\": *" + }, + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "counter_rate", + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "cumulative_sum", + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X4" + ], + "scale": "ratio" + }, + "56e40393-65fa-403e-b757-05ae1fc04063X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events processed/ms", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "56e40393-65fa-403e-b757-05ae1fc04063X2", + "56e40393-65fa-403e-b757-05ae1fc04063X5" + ], + "location": { + "max": 174, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.time.duration.ms)))", + "type": "function" + } + }, + "references": [ + "56e40393-65fa-403e-b757-05ae1fc04063X2", + "56e40393-65fa-403e-b757-05ae1fc04063X5" + ], + "scale": "ratio" + }, + "5f2a9ea0-38a8-403a-bf56-5792745b0b7a": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Worker millis per event", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "65ca78d2-d895-4db6-bec3-7e110bd0030b": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.input.events.out: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.filter.name" + }, + "6fdcaadf-6c83-43f1-9f65-b148037b7aab": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Worker utilization", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 3 + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute" + }, + "8914109f-497d-4293-8692-dbdddffbf42f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events received", + "operationType": "counter_rate", + "references": [ + "b1988bb4-4923-4f92-b63e-1cc9d230457f" + ], + "scale": "ratio" + }, + "b1988bb4-4923-4f92-b63e-1cc9d230457f": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.plugin.filter.events.in", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.in" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbe": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events emitted", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.filter.events.out))", + "isFormulaBroken": false + }, + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX1" + ], + "scale": "ratio" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.out" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted", + "operationType": "counter_rate", + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX0" + ], + "scale": "ratio", + "timeScale": "s" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "collapseFn": "sum", + "columnId": "410757d6-0433-4cdd-b51b-138704dfc00a", + "hidden": true, + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "0a5be802-a381-4edd-ad8c-488a6a419619", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "d172976a-5c43-46b7-9d96-44ac40134cbe", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "3bdc4103-5313-4e00-a407-de5b7fde820c", + "isTransposed": false, + "palette": { + "name": "custom", + "params": { + "colorStops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ], + "continuity": "above", + "name": "custom", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#209280", + "stop": 20 + }, + { + "color": "#54b399", + "stop": 40 + }, + { + "color": "#d6bf57", + "stop": 60 + }, + { + "color": "#e7664c", + "stop": 80 + }, + { + "color": "#cc5642", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "8914109f-497d-4293-8692-dbdddffbf42f", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "5f2a9ea0-38a8-403a-bf56-5792745b0b7a", + "isTransposed": false, + "palette": { + "name": "custom", + "params": { + "colorStops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ], + "continuity": "above", + "name": "custom", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#209280", + "stop": 20 + }, + { + "color": "#54b399", + "stop": 40 + }, + { + "color": "#d6bf57", + "stop": 60 + }, + { + "color": "#e7664c", + "stop": 80 + }, + { + "color": "#cc5642", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "6fdcaadf-6c83-43f1-9f65-b148037b7aab", + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "56e40393-65fa-403e-b757-05ae1fc04063", + "isTransposed": false, + "palette": { + "name": "custom", + "params": { + "colorStops": [ + { + "color": "#cc5642", + "stop": 0 + }, + { + "color": "#e7664c", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#54b399", + "stop": 60 + }, + { + "color": "#209280", + "stop": 80 + } + ], + "continuity": "above", + "name": "custom", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#cc5642", + "stop": 20 + }, + { + "color": "#e7664c", + "stop": 40 + }, + { + "color": "#d6bf57", + "stop": 60 + }, + { + "color": "#54b399", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "columnId": "18e039ef-9926-4559-bb1a-d2894c21f750", + "isTransposed": false + } + ], + "layerId": "df79554f-08e3-4f7f-b438-80bc81e14637", + "layerType": "data", + "sorting": { + "columnId": "6fdcaadf-6c83-43f1-9f65-b148037b7aab", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 14, + "i": "97bcb728-f0e5-4047-aade-2ef56bd33c22", + "w": 40, + "x": 8, + "y": 56 + }, + "panelIndex": "97bcb728-f0e5-4047-aade-2ef56bd33c22", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.time.duration.ms\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.filter.name", + "logstash.pipeline.plugin.filter.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Milliseconds per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "\ncumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.plugin.filter.events.in))))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "cumulative_sum", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.in" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "cumulative_sum", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" + ], + "location": { + "max": 163, + "min": 0 + }, + "name": "divide", + "text": "\ncumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.plugin.filter.events.in))))", + "type": "function" + } + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" + ], + "scale": "ratio" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 10, + "i": "f8033492-7353-4564-83d5-6715e387f9ac", + "w": 40, + "x": 8, + "y": 70 + }, + "panelIndex": "f8033492-7353-4564-83d5-6715e387f9ac", + "title": "Average time spent processing each event", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.filter.name", + "logstash.pipeline.plugin.filter.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Received per second", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.in))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.events.in\": *" + }, + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.in" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "d6db9db8-7da0-435c-8118-9d0c36d2573d", + "w": 20, + "x": 8, + "y": 80 + }, + "panelIndex": "d6db9db8-7da0-435c-8118-9d0c36d2573d", + "title": "Events Received per second", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.filter.name", + "logstash.pipeline.plugin.filter.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Emitted per second", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.out))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.events.out\": *" + }, + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.events.out" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", + "w": 20, + "x": 28, + "y": 80 + }, + "panelIndex": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", + "title": "Events Emitted per second", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Worker Utilization", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "ff88ade5-20cd-4210-b34e-7712c5da3acb": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", + "w": 20, + "x": 8, + "y": 91 + }, + "panelIndex": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", + "title": "Worker Utilization", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "ebc7c60d-e028-4662-9301-3f878b43f606" + ], + "columns": { + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Worker Millis per event", + "operationType": "counter_rate", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + }, + "references": [ + "ebc7c60d-e028-4662-9301-3f878b43f606" + ], + "scale": "ratio" + }, + "ebc7c60d-e028-4662-9301-3f878b43f606": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "ff88ade5-20cd-4210-b34e-7712c5da3acb": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.filter.name", + "logstash.pipeline.plugin.filter.id" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "48f64446-f5ca-451d-8a59-bb12b4807381", + "w": 20, + "x": 28, + "y": 91 + }, + "panelIndex": "48f64446-f5ca-451d-8a59-bb12b4807381", + "title": "Worker Milliseconds per event", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "Outputs", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "cfd22405-6008-4373-ad69-94e70427e259", + "w": 24, + "x": 17, + "y": 102 + }, + "panelIndex": "cfd22405-6008-4373-ad69-94e70427e259", + "title": "Plugins", + "type": "visualization", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "df79554f-08e3-4f7f-b438-80bc81e14637": { + "columnOrder": [ + "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "5cbd518c-9c32-454d-88a3-935dab1a7fda", + "0a5be802-a381-4edd-ad8c-488a6a419619", + "410757d6-0433-4cdd-b51b-138704dfc00a", + "125a425e-8c72-4318-96aa-ee1fb0edc521", + "d172976a-5c43-46b7-9d96-44ac40134cbe", + "e0728321-e2b4-44c6-b07b-a89f632f02e1", + "3bdc4103-5313-4e00-a407-de5b7fde820c", + "a7134cf0-c941-41e8-b346-40bc893c2514", + "8cb013fb-a55f-4c6d-88e1-6ce798c8db07", + "d172976a-5c43-46b7-9d96-44ac40134cbeX0", + "d172976a-5c43-46b7-9d96-44ac40134cbeX1", + "3bdc4103-5313-4e00-a407-de5b7fde820cX0", + "3bdc4103-5313-4e00-a407-de5b7fde820cX1", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X0", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X1", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X2", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X3", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X4", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X5", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X6", + "125a425e-8c72-4318-96aa-ee1fb0edc521X0", + "125a425e-8c72-4318-96aa-ee1fb0edc521X1" + ], + "columns": { + "0a5be802-a381-4edd-ad8c-488a6a419619": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Id", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.events.out: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.output.id" + }, + "125a425e-8c72-4318-96aa-ee1fb0edc521": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Received", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.output.events.in))", + "isFormulaBroken": false + }, + "references": [ + "125a425e-8c72-4318-96aa-ee1fb0edc521X1" + ], + "scale": "ratio" + }, + "125a425e-8c72-4318-96aa-ee1fb0edc521X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Received", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.in" + }, + "125a425e-8c72-4318-96aa-ee1fb0edc521X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Received", + "operationType": "counter_rate", + "references": [ + "125a425e-8c72-4318-96aa-ee1fb0edc521X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent in plugin", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.output.time.duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + ], + "scale": "ratio" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent in plugin", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.time.duration.ms" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent in plugin", + "operationType": "counter_rate", + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "410757d6-0433-4cdd-b51b-138704dfc00a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "5cbd518c-9c32-454d-88a3-935dab1a7fda": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "8cb013fb-a55f-4c6d-88e1-6ce798c8db07", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "65ca78d2-d895-4db6-bec3-7e110bd0030b": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a7134cf0-c941-41e8-b346-40bc893c2514", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.output.name" + }, + "8cb013fb-a55f-4c6d-88e1-6ce798c8db07": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Worker utilization", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + }, + "a7134cf0-c941-41e8-b346-40bc893c2514": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Worker millis per event", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbe": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events emitted", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.output.events.out))", + "isFormulaBroken": false + }, + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX1" + ], + "scale": "ratio" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events emitted", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.out" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events emitted", + "operationType": "counter_rate", + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events output (per ms)", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "/ms" + } + }, + "formula": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.time.duration.ms)))", + "isFormulaBroken": false + }, + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X6" + ], + "scale": "ratio" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.events.in\": *" + }, + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.in" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "counter_rate", + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "cumulative_sum", + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X1" + ], + "scale": "ratio" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X3": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.time.duration.ms\": *" + }, + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.time.duration.ms" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "counter_rate", + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "cumulative_sum", + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X4" + ], + "scale": "ratio" + }, + "e0728321-e2b4-44c6-b07b-a89f632f02e1X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events output (per ms)", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X2", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X5" + ], + "location": { + "max": 174, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.time.duration.ms)))", + "type": "function" + } + }, + "references": [ + "e0728321-e2b4-44c6-b07b-a89f632f02e1X2", + "e0728321-e2b4-44c6-b07b-a89f632f02e1X5" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "collapseFn": "sum", + "columnId": "410757d6-0433-4cdd-b51b-138704dfc00a", + "hidden": true, + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "0a5be802-a381-4edd-ad8c-488a6a419619", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "d172976a-5c43-46b7-9d96-44ac40134cbe", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "3bdc4103-5313-4e00-a407-de5b7fde820c", + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "a7134cf0-c941-41e8-b346-40bc893c2514", + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "8cb013fb-a55f-4c6d-88e1-6ce798c8db07", + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "columnId": "5cbd518c-9c32-454d-88a3-935dab1a7fda", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "e0728321-e2b4-44c6-b07b-a89f632f02e1", + "isTransposed": false, + "palette": { + "name": "custom", + "params": { + "colorStops": [ + { + "color": "#cc5642", + "stop": 0 + }, + { + "color": "#e7664c", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#54b399", + "stop": 60 + }, + { + "color": "#209280", + "stop": 80 + } + ], + "continuity": "above", + "name": "custom", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#cc5642", + "stop": 20 + }, + { + "color": "#e7664c", + "stop": 40 + }, + { + "color": "#d6bf57", + "stop": 60 + }, + { + "color": "#54b399", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "125a425e-8c72-4318-96aa-ee1fb0edc521", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + } + ], + "layerId": "df79554f-08e3-4f7f-b438-80bc81e14637", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 16, + "i": "5ea51e10-bede-40f6-9842-f840617bcc8b", + "w": 40, + "x": 8, + "y": 106 + }, + "panelIndex": "5ea51e10-bede-40f6-9842-f840617bcc8b", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.time.duration.ms\": *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Milliseconds per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "\ncumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.plugin.output.events.out))))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.time.duration.ms" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "cumulative_sum", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.out" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "cumulative_sum", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX4" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Milliseconds per event", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" + ], + "location": { + "max": 164, + "min": 0 + }, + "name": "divide", + "text": "\ncumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.plugin.output.events.out))))", + "type": "function" + } + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" + ], + "scale": "ratio" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "bb1876b7-7faa-4c37-a8af-c79a70149922", + "w": 40, + "x": 8, + "y": 122 + }, + "panelIndex": "bb1876b7-7faa-4c37-a8af-c79a70149922", + "title": "Average time spent processing event (ms)", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events received (/s)", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(last_value(logstash.pipeline.plugin.output.events.in))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.events.in\": *" + }, + "isBucketed": false, + "label": "Part of Events received per second", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.in" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events received per second", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "c49e6ec7-e4a6-4938-b39b-d0c49b1ae114", + "w": 20, + "x": 8, + "y": 133 + }, + "panelIndex": "c49e6ec7-e4a6-4938-b39b-d0c49b1ae114", + "title": "Events received per second", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events emitted (/s)", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(last_value(logstash.pipeline.plugin.output.events.out))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.output.events.out\": *" + }, + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.events.out" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted per second", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "palette": { + "name": "default", + "type": "palette" + }, + "seriesType": "line", + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "231824a2-a183-439f-8e08-e1bad6c5e9db", + "w": 20, + "x": 28, + "y": 133 + }, + "panelIndex": "231824a2-a183-439f-8e08-e1bad6c5e9db", + "title": "Events emitted per second", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Worker Millis per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "ff88ade5-20cd-4210-b34e-7712c5da3acb": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "09cd3d58-f4a5-4033-a011-15002b6649e7", + "w": 20, + "x": 8, + "y": 144 + }, + "panelIndex": "09cd3d58-f4a5-4033-a011-15002b6649e7", + "title": "Worker millseconds per event", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { + "columnOrder": [ + "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "columns": { + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Worker Utilization", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "ff88ade5-20cd-4210-b34e-7712c5da3acb": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "03d92828-8549-4cd9-a7ea-12f25502cef1", + "w": 20, + "x": 28, + "y": 144 + }, + "panelIndex": "03d92828-8549-4cd9-a7ea-12f25502cef1", + "title": "Worker utilization", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "description": "Codecs", + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "Codecs", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "31278c59-d923-4beb-800b-a8107e9064c5", + "w": 24, + "x": 16, + "y": 155 + }, + "panelIndex": "31278c59-d923-4beb-800b-a8107e9064c5", + "title": "Plugins", + "type": "visualization", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "df79554f-08e3-4f7f-b438-80bc81e14637": { + "columnOrder": [ + "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "19f6617a-a8fe-4dfc-9189-b10738677289", + "0a5be802-a381-4edd-ad8c-488a6a419619", + "410757d6-0433-4cdd-b51b-138704dfc00a", + "d172976a-5c43-46b7-9d96-44ac40134cbe", + "3bdc4103-5313-4e00-a407-de5b7fde820c", + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7", + "9f77bcc7-661e-437e-a8c8-8f004dd80b64", + "64c03b15-ed7f-4774-a04e-238b21d9c850", + "d172976a-5c43-46b7-9d96-44ac40134cbeX0", + "d172976a-5c43-46b7-9d96-44ac40134cbeX1", + "3bdc4103-5313-4e00-a407-de5b7fde820cX0", + "3bdc4103-5313-4e00-a407-de5b7fde820cX1", + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X0", + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X1", + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X0", + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X1", + "64c03b15-ed7f-4774-a04e-238b21d9c850X0", + "64c03b15-ed7f-4774-a04e-238b21d9c850X1" + ], + "columns": { + "0a5be802-a381-4edd-ad8c-488a6a419619": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Id", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.events.out: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.time.duration.ms", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.codec.id" + }, + "19f6617a-a8fe-4dfc-9189-b10738677289": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 20 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent encoding", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.codec.encode.duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + ], + "scale": "ratio" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time Spent Encoding", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.codec.encode.duration.ms" + }, + "3bdc4103-5313-4e00-a407-de5b7fde820cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time Spent Encoding", + "operationType": "counter_rate", + "references": [ + "3bdc4103-5313-4e00-a407-de5b7fde820cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "410757d6-0433-4cdd-b51b-138704dfc00a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "64c03b15-ed7f-4774-a04e-238b21d9c850": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent decoding", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "ms" + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.codec.decode.duration.ms))", + "isFormulaBroken": false + }, + "references": [ + "64c03b15-ed7f-4774-a04e-238b21d9c850X1" + ], + "scale": "ratio" + }, + "64c03b15-ed7f-4774-a04e-238b21d9c850X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time Spent Decoding", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.codec.decode.duration.ms" + }, + "64c03b15-ed7f-4774-a04e-238b21d9c850X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time Spent Decoding", + "operationType": "counter_rate", + "references": [ + "64c03b15-ed7f-4774-a04e-238b21d9c850X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "65ca78d2-d895-4db6-bec3-7e110bd0030b": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.codec.name" + }, + "9f77bcc7-661e-437e-a8c8-8f004dd80b64": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events decoded (out)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.codec.decode.out))", + "isFormulaBroken": false + }, + "references": [ + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X1" + ], + "scale": "ratio" + }, + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Decoded Out", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.codec.decode.out" + }, + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Decoded Out", + "operationType": "counter_rate", + "references": [ + "9f77bcc7-661e-437e-a8c8-8f004dd80b64X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events decoded (in)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.codec.decode.in))", + "isFormulaBroken": false + }, + "references": [ + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X1" + ], + "scale": "ratio" + }, + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Decoded In", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.codec.decode.in" + }, + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Decoded In", + "operationType": "counter_rate", + "references": [ + "b0e1236c-4e69-4e05-834f-5382ba5dc4f7X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbe": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events encoded", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.plugin.codec.encode.in))", + "isFormulaBroken": false + }, + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX1" + ], + "scale": "ratio" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Encoded", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.codec.encode.in" + }, + "d172976a-5c43-46b7-9d96-44ac40134cbeX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Encoded", + "operationType": "counter_rate", + "references": [ + "d172976a-5c43-46b7-9d96-44ac40134cbeX0" + ], + "scale": "ratio", + "timeScale": "s" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "collapseFn": "sum", + "columnId": "410757d6-0433-4cdd-b51b-138704dfc00a", + "hidden": true, + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "0a5be802-a381-4edd-ad8c-488a6a419619", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "d172976a-5c43-46b7-9d96-44ac40134cbe", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "3bdc4103-5313-4e00-a407-de5b7fde820c", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "b0e1236c-4e69-4e05-834f-5382ba5dc4f7", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "9f77bcc7-661e-437e-a8c8-8f004dd80b64", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "64c03b15-ed7f-4774-a04e-238b21d9c850", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "columnId": "19f6617a-a8fe-4dfc-9189-b10738677289", + "isTransposed": false + } + ], + "layerId": "df79554f-08e3-4f7f-b438-80bc81e14637", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 12, + "i": "46fbd106-9dba-4c2c-b359-fd04c22e1314", + "w": 40, + "x": 8, + "y": 159 + }, + "panelIndex": "46fbd106-9dba-4c2c-b359-fd04c22e1314", + "type": "lens", + "version": "8.10.1" + } + ], + "timeRestore": false, + "title": "[Metrics Logstash] Logstash Plugins - Advanced View", + "version": 1 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T19:44:30.925Z", + "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", + "managed": false, + "references": [ + { + "id": "logstash-sm-metrics", + "name": "9a756479-5952-441e-beff-e0896500ca39:indexpattern-datasource-layer-6cc6ec80-cf6c-4421-a323-256f08b59426", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "9a756479-5952-441e-beff-e0896500ca39:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "6da9466a-ede0-44b6-a2e7-d704065bb90b:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "f03395de-a163-4204-ac82-83c0ea542eeb:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "4fad1f03-99d3-4834-a6c4-bea283178567:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "a3444242-6e3e-4959-8815-43e65b3e8d69:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "5171ef08-d301-4a9c-832b-40f41c9ad24d:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "9a34af7e-5f12-49b4-86e0-5577203711eb:indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "f94ee279-63d8-45de-bfa0-9c93416b9752:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "1eabaf75-95dd-4cae-95d5-f00efe978872:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "3470dd76-7ebd-4b19-a3de-fa0082a72ee5:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "97bcb728-f0e5-4047-aade-2ef56bd33c22:indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "f8033492-7353-4564-83d5-6715e387f9ac:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "d6db9db8-7da0-435c-8118-9d0c36d2573d:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "48f64446-f5ca-451d-8a59-bb12b4807381:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "5ea51e10-bede-40f6-9842-f840617bcc8b:indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "bb1876b7-7faa-4c37-a8af-c79a70149922:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "c49e6ec7-e4a6-4938-b39b-d0c49b1ae114:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "231824a2-a183-439f-8e08-e1bad6c5e9db:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "09cd3d58-f4a5-4033-a011-15002b6649e7:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "03d92828-8549-4cd9-a7ea-12f25502cef1:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "46fbd106-9dba-4c2c-b359-fd04c22e1314:indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_429cdb8b-f9d9-45b0-bf04-2fca76ad3cd8:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_497f4765-376b-4a02-9982-0e8bf9a0b1ef:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "8.9.0" +} \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json new file mode 100644 index 00000000000..b682b5d8493 --- /dev/null +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -0,0 +1,1612 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"94cc8a2a-a81e-451b-891b-407075069331\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"94cc8a2a-a81e-451b-891b-407075069331\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{},\"selectedOptions\":[]}},\"73fdfb3a-3e86-499a-93f1-993479254e4e\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"73fdfb3a-3e86-499a-93f1-993479254e4e\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" + }, + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-cf96630a-a4c7-4bb8-9a5e-25241521dea6", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "cf96630a-a4c7-4bb8-9a5e-25241521dea6": { + "columnOrder": [ + "663eeedf-a114-49b8-b9b4-90d963d07509", + "663eeedf-a114-49b8-b9b4-90d963d07509X0" + ], + "columns": { + "663eeedf-a114-49b8-b9b4-90d963d07509": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Pipelines running", + "operationType": "formula", + "params": { + "format": { + "id": "custom", + "params": { + "decimals": 0 + } + }, + "formula": "unique_count(logstash.pipeline.name)", + "isFormulaBroken": false + }, + "reducedTimeRange": "30s", + "references": [ + "663eeedf-a114-49b8-b9b4-90d963d07509X0" + ], + "scale": "ratio" + }, + "663eeedf-a114-49b8-b9b4-90d963d07509X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Pipelines Running", + "operationType": "unique_count", + "params": { + "emptyAsNull": false + }, + "reducedTimeRange": "30s", + "scale": "ratio", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "icon": "empty", + "layerId": "cf96630a-a4c7-4bb8-9a5e-25241521dea6", + "layerType": "data", + "metricAccessor": "663eeedf-a114-49b8-b9b4-90d963d07509", + "showBar": false, + "trendlineSecondaryMetricAccessor": "83c85588-e89d-4b55-8677-23f339dba50c" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "5f909278-92f5-4ebc-bb18-8aca8f987733", + "w": 10, + "x": 8, + "y": 0 + }, + "panelIndex": "5f909278-92f5-4ebc-bb18-8aca8f987733", + "title": "", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "69a27ae4-5987-429d-89e1-84e7fb1b8af8": { + "columnOrder": [ + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f" + ], + "columns": { + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Logstash instances running pipelines", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "reducedTimeRange": "30s", + "scale": "ratio", + "sourceField": "logstash.pipeline.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "layerType": "data", + "metricAccessor": "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f", + "showBar": false + } + }, + "title": "[Metrics Logstash] Nodes running pipelines viz (copy)", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "239e17fa-3741-4bd5-a44c-6488c53c7c2b", + "w": 10, + "x": 18, + "y": 0 + }, + "panelIndex": "239e17fa-3741-4bd5-a44c-6488c53c7c2b", + "title": "[Metrics Logstash] Nodes running pipelines viz", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "69a27ae4-5987-429d-89e1-84e7fb1b8af8": { + "columnOrder": [ + "09c8e9b3-3be7-4858-82d4-e54ababafc70", + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f" + ], + "columns": { + "09c8e9b3-3be7-4858-82d4-e54ababafc70": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + }, + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Total number of events received", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "09c8e9b3-3be7-4858-82d4-e54ababafc70", + "collapseFn": "sum", + "layerId": "69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "layerType": "data", + "metricAccessor": "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f", + "showBar": false + } + }, + "title": "[Metrics Logstash] Total Events Received viz ", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "0134bb0e-0e1d-49ec-92d4-d775c64c7bdb", + "w": 10, + "x": 28, + "y": 0 + }, + "panelIndex": "0134bb0e-0e1d-49ec-92d4-d775c64c7bdb", + "title": "[Metrics Logstash] Total Events Received viz ", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "69a27ae4-5987-429d-89e1-84e7fb1b8af8": { + "columnOrder": [ + "09c8e9b3-3be7-4858-82d4-e54ababafc70", + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f" + ], + "columns": { + "09c8e9b3-3be7-4858-82d4-e54ababafc70": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + }, + "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.out: *" + }, + "isBucketed": false, + "label": "Total number of events emitted", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "09c8e9b3-3be7-4858-82d4-e54ababafc70", + "collapseFn": "sum", + "layerId": "69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "layerType": "data", + "metricAccessor": "fb3c93e3-0cdd-404f-aa92-588ea2a6e76f", + "showBar": false + } + }, + "title": "[Metrics Logstash] Total Events Emitted viz (copy)", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "8199ac33-4d5b-46e0-b3cd-3683204cc65c", + "w": 10, + "x": 38, + "y": 0 + }, + "panelIndex": "8199ac33-4d5b-46e0-b3cd-3683204cc65c", + "title": "[Metrics Logstash] Total Events Emitted viz", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-af02f11d-c2e6-46a1-87fc-76139bbe0998", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "af02f11d-c2e6-46a1-87fc-76139bbe0998": { + "columnOrder": [ + "2b095037-5d7d-4cb4-a265-afb041616816", + "2d6ddf9d-fd47-4ec3-8e3c-fad74fbff7f1", + "8b340f82-b329-4adc-8a42-82c2a8495d2c", + "7f9470fb-8d50-485f-a388-9c18afe3e735", + "1c8d484c-64cf-4497-bede-36e69ad4047f", + "5af7d5b3-3989-4473-89e7-17461f4b9c22", + "d1d9a76c-a44f-4027-853e-263043f202a1", + "cc68d659-2c7d-4689-b30a-c536d451cf0f" + ], + "columns": { + "1c8d484c-64cf-4497-bede-36e69ad4047f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent in pipeline", + "operationType": "counter_rate", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + } + }, + "references": [ + "d1d9a76c-a44f-4027-853e-263043f202a1" + ], + "scale": "ratio" + }, + "2b095037-5d7d-4cb4-a265-afb041616816": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "7f9470fb-8d50-485f-a388-9c18afe3e735", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + }, + "2d6ddf9d-fd47-4ec3-8e3c-fad74fbff7f1": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "5af7d5b3-3989-4473-89e7-17461f4b9c22": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent pushing to queue", + "operationType": "counter_rate", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + } + }, + "references": [ + "cc68d659-2c7d-4689-b30a-c536d451cf0f" + ], + "scale": "ratio" + }, + "7f9470fb-8d50-485f-a388-9c18afe3e735": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events emitted", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "8b340f82-b329-4adc-8a42-82c2a8495d2c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events received", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + }, + "cc68d659-2c7d-4689-b30a-c536d451cf0f": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.time.queue_push_duration.ms", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.time.queue_push_duration.ms" + }, + "d1d9a76c-a44f-4027-853e-263043f202a1": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.time.duration.ms", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.time.duration.ms" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "columnId": "2b095037-5d7d-4cb4-a265-afb041616816", + "isTransposed": false + }, + { + "collapseFn": "sum", + "columnId": "2d6ddf9d-fd47-4ec3-8e3c-fad74fbff7f1", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "8b340f82-b329-4adc-8a42-82c2a8495d2c", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "7f9470fb-8d50-485f-a388-9c18afe3e735", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "1c8d484c-64cf-4497-bede-36e69ad4047f", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "5af7d5b3-3989-4473-89e7-17461f4b9c22", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + } + ], + "layerId": "af02f11d-c2e6-46a1-87fc-76139bbe0998", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": true, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "Go to Dashboard" + }, + "eventId": "ace863df-5d1f-415c-a5d0-cecc3e9beedd", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "7f8d9886-3037-425d-bcb4-ae92a2b0d16e", + "w": 40, + "x": 8, + "y": 5 + }, + "panelIndex": "7f8d9886-3037-425d-bcb4-ae92a2b0d16e", + "title": "Pipelines", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "0dc38ed3-eda8-48d3-85d1-454d187be2f1": { + "columnOrder": [ + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "66ce1290-0494-4de1-b877-e672f10deaf3", + "735d813f-2024-473c-bc2c-69cbcfb5ff29", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + ], + "columns": { + "66ce1290-0494-4de1-b877-e672f10deaf3": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events Emitted Rate", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.total.events.out), kql='logstash.pipeline.total.events.in: *')", + "isFormulaBroken": false + }, + "references": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Emitted Rate", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Part of Events Emitted Rate", + "operationType": "counter_rate", + "references": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "layers": [ + { + "accessors": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29" + ], + "layerId": "0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "xAccessor": "66ce1290-0494-4de1-b877-e672f10deaf3" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", + "w": 20, + "x": 8, + "y": 12 + }, + "panelIndex": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", + "title": "Events Emitted Rate ", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "0dc38ed3-eda8-48d3-85d1-454d187be2f1": { + "columnOrder": [ + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "66ce1290-0494-4de1-b877-e672f10deaf3", + "735d813f-2024-473c-bc2c-69cbcfb5ff29", + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + ], + "columns": { + "66ce1290-0494-4de1-b877-e672f10deaf3": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Events Received Rate", + "operationType": "counter_rate", + "references": [ + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.events.in", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + }, + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "layers": [ + { + "accessors": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29" + ], + "layerId": "0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "xAccessor": "66ce1290-0494-4de1-b877-e672f10deaf3" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "1a42a8c2-9823-4178-90ae-f3a403b63711", + "w": 20, + "x": 28, + "y": 12 + }, + "panelIndex": "1a42a8c2-9823-4178-90ae-f3a403b63711", + "title": "Events Received Rate", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "0dc38ed3-eda8-48d3-85d1-454d187be2f1": { + "columnOrder": [ + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "66ce1290-0494-4de1-b877-e672f10deaf3", + "735d813f-2024-473c-bc2c-69cbcfb5ff29", + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + ], + "columns": { + "66ce1290-0494-4de1-b877-e672f10deaf3": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Persistent Queue Event Count", + "operationType": "counter_rate", + "references": [ + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + ], + "scale": "ratio" + }, + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.pipeline.total.queues.events", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.events" + }, + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "layers": [ + { + "accessors": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29" + ], + "layerId": "0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "xAccessor": "66ce1290-0494-4de1-b877-e672f10deaf3" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "4017b8f5-f8f2-46b9-86fe-501f46d551c3", + "w": 20, + "x": 8, + "y": 19 + }, + "panelIndex": "4017b8f5-f8f2-46b9-86fe-501f46d551c3", + "title": "Persistent Queue Event Count", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "43c6d3b9-ff0a-40c8-bc5b-07585d2acf47", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "0dc38ed3-eda8-48d3-85d1-454d187be2f1": { + "columnOrder": [ + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "66ce1290-0494-4de1-b877-e672f10deaf3", + "735d813f-2024-473c-bc2c-69cbcfb5ff29", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X2" + ], + "columns": { + "66ce1290-0494-4de1-b877-e672f10deaf3": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Persistent Queue Event Count", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "compact": true, + "decimals": 2 + } + }, + "formula": "round(last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes), 2)", + "isFormulaBroken": false + }, + "references": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X2" + ], + "scale": "ratio" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.queues.current_size.bytes\": *" + }, + "isBucketed": false, + "label": "Part of Persistent Queue Event Count", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.current_size.bytes" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.queues.max_size.bytes\": *" + }, + "isBucketed": false, + "label": "Part of Persistent Queue Event Count", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.max_size.bytes" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Persistent Queue Event Count", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + { + "args": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + ], + "location": { + "max": 125, + "min": 6 + }, + "name": "divide", + "text": "last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes)", + "type": "function" + }, + 2 + ], + "location": { + "max": 129, + "min": 0 + }, + "name": "round", + "text": "round(last_value(logstash.pipeline.total.queues.current_size.bytes)/last_value(logstash.pipeline.total.queues.max_size.bytes), 2)", + "type": "function" + } + }, + "references": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + ], + "scale": "ratio" + }, + "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.total.queues.type", + "index": "43c6d3b9-ff0a-40c8-bc5b-07585d2acf47", + "key": "logstash.pipeline.total.queues.type", + "negate": false, + "params": { + "query": "persisted" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.total.queues.type": "persisted" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "layers": [ + { + "accessors": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29" + ], + "layerId": "0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", + "xAccessor": "66ce1290-0494-4de1-b877-e672f10deaf3" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide", + "yLeftExtent": { + "lowerBound": 0, + "mode": "custom", + "upperBound": 1 + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "d290b2f1-1769-41f4-9d29-41755eef8871", + "w": 20, + "x": 28, + "y": 19 + }, + "panelIndex": "d290b2f1-1769-41f4-9d29-41755eef8871", + "title": "Persistent Queue Disk Utilization", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n**[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548)** \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\n\nOverview\n\nThis Dashboard gives an overall view of Pipelines Running on Logstash", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 26, + "i": "3edb2e9f-2807-4e65-9adc-259c15debce9", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "3edb2e9f-2807-4e65-9adc-259c15debce9", + "title": "Logstash Nav Panel", + "type": "visualization", + "version": "8.10.1" + } + ], + "timeRestore": false, + "title": "[Metrics Logstash] Pipelines Overview", + "version": 1 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T19:05:10.115Z", + "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", + "managed": false, + "references": [ + { + "id": "logstash-sm-metrics", + "name": "5f909278-92f5-4ebc-bb18-8aca8f987733:indexpattern-datasource-layer-cf96630a-a4c7-4bb8-9a5e-25241521dea6", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "239e17fa-3741-4bd5-a44c-6488c53c7c2b:indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "0134bb0e-0e1d-49ec-92d4-d775c64c7bdb:indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "8199ac33-4d5b-46e0-b3cd-3683204cc65c:indexpattern-datasource-layer-69a27ae4-5987-429d-89e1-84e7fb1b8af8", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "7f8d9886-3037-425d-bcb4-ae92a2b0d16e:indexpattern-datasource-layer-af02f11d-c2e6-46a1-87fc-76139bbe0998", + "type": "index-pattern" + }, + { + "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", + "name": "7f8d9886-3037-425d-bcb4-ae92a2b0d16e:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:ace863df-5d1f-415c-a5d0-cecc3e9beedd:dashboardId", + "type": "dashboard" + }, + { + "id": "logstash-sm-metrics", + "name": "09177cfd-5361-4e91-b7c3-64eeb7837ce8:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "1a42a8c2-9823-4178-90ae-f3a403b63711:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "4017b8f5-f8f2-46b9-86fe-501f46d551c3:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "d290b2f1-1769-41f4-9d29-41755eef8871:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "d290b2f1-1769-41f4-9d29-41755eef8871:43c6d3b9-ff0a-40c8-bc5b-07585d2acf47", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_94cc8a2a-a81e-451b-891b-407075069331:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_73fdfb3a-3e86-499a-93f1-993479254e4e:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "8.9.0" +} \ No newline at end of file From 55b7bb15cf4b976714b4724c1053a0a5e34de39d Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 10:15:14 -0400 Subject: [PATCH 05/25] Update existing dashboards to include pipeline and plugin references --- ...-79270240-48ee-11ee-8cb5-99927777c522.json | 1150 +++++++++-------- ...-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 288 ++++- ...-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 765 ++++++++--- ...-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 51 +- 4 files changed, 1514 insertions(+), 740 deletions(-) diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index fa2e4d23198..c2f399acf3d 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1,5 +1,11 @@ { "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"5c0bd4d8-47ca-4a2f-a1ff-d9f4b55dfad0\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"5c0bd4d8-47ca-4a2f-a1ff-d9f4b55dfad0\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"c5cca387-507e-44a6-883f-0ab948c24319\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"c5cca387-507e-44a6-883f-0ab948c24319\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" + }, "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { @@ -37,7 +43,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n**[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522)** \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\nOverview\n\nThis Dashboard gives an overall view of Logstash", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n**[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522)** \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\nOverview\n\nThis Dashboard gives an overall view of Logstash", "openLinksInNewTab": false }, "title": "", @@ -54,7 +60,7 @@ }, "panelIndex": "67c48168-cf30-4dcb-a96e-8e0a38e6049d", "type": "visualization", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -62,7 +68,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", + "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "type": "index-pattern" } ], @@ -71,25 +77,82 @@ "datasourceStates": { "formBased": { "layers": { - "3eb7cb2b-67b2-4fc7-b311-9106826dbbfc": { + "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { "columnOrder": [ - "a45f2761-e598-4ec1-8dff-95b395d0dd1e" + "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "0c423bdf-578e-4f0a-bb27-180a24a133e1", + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" ], "columns": { - "a45f2761-e598-4ec1-8dff-95b395d0dd1e": { + "0c423bdf-578e-4f0a-bb27-180a24a133e1": { "customLabel": true, "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.events.in : *" + }, "isBucketed": false, - "label": " ", - "operationType": "unique_count", + "label": "Events Emitted Rate", + "operationType": "counter_rate", + "references": [ + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "scale": "ratio", + "timeScale": "s" + }, + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.node.stats.events.out", + "operationType": "max", "params": { "emptyAsNull": true }, "scale": "ratio", + "sourceField": "logstash.node.stats.events.out" + }, + "ef4e5445-487e-4a0c-ac01-063e8c199a84": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.node.stats.logstash.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", "sourceField": "logstash.node.stats.logstash.name" } }, - "incompleteColumns": {} + "incompleteColumns": {}, + "sampling": 1 } } }, @@ -104,29 +167,80 @@ "query": "" }, "visualization": { - "accessor": "a45f2761-e598-4ec1-8dff-95b395d0dd1e", - "layerId": "3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", - "layerType": "data" + "axisTitlesVisibilitySettings": { + "x": false, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "hideEndzones": true, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1" + ], + "collapseFn": "sum", + "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } } }, - "title": "Nodes", + "title": "", "type": "lens", - "visualizationType": "lnsLegacyMetric" + "visualizationType": "lnsXY" }, "enhancements": {}, "hidePanelTitles": false }, "gridData": { - "h": 4, - "i": "e7bf7bee-710a-456a-995c-a570ee24cc05", - "w": 10, + "h": 10, + "i": "a66223a5-9fdb-4335-8012-4ae2748928ac", + "w": 20, "x": 8, - "y": 0 + "y": 4 }, - "panelIndex": "e7bf7bee-710a-456a-995c-a570ee24cc05", - "title": "Nodes", + "panelIndex": "a66223a5-9fdb-4335-8012-4ae2748928ac", + "title": "Events Emitted Rate/s", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -134,7 +248,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-d944a0ae-9030-413e-afd6-c4a57ff34ad0", + "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "type": "index-pattern" } ], @@ -143,35 +257,55 @@ "datasourceStates": { "formBased": { "layers": { - "d944a0ae-9030-413e-afd6-c4a57ff34ad0": { + "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { "columnOrder": [ - "b1f83d95-ed15-4e15-8e1e-b912424a35b2", - "a98fb979-a039-4322-871f-1d32480fd158" + "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "0c423bdf-578e-4f0a-bb27-180a24a133e1", + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" ], "columns": { - "a98fb979-a039-4322-871f-1d32480fd158": { + "0c423bdf-578e-4f0a-bb27-180a24a133e1": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.jvm.mem.heap_used_in_bytes: *" + "query": "logstash.node.stats.events.in : *" }, "isBucketed": false, - "label": "Heap Used", - "operationType": "last_value", + "label": "Events Received Rate", + "operationType": "counter_rate", + "references": [ + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "scale": "ratio", + "timeScale": "s" + }, + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", "params": { - "format": { - "id": "bytes", - "params": { - "decimals": 2 - } - }, - "sortField": "@timestamp" + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.node.stats.events.in", + "operationType": "max", + "params": { + "emptyAsNull": true }, "scale": "ratio", - "sourceField": "logstash.node.stats.jvm.mem.heap_used_in_bytes" + "sourceField": "logstash.node.stats.events.in" }, - "b1f83d95-ed15-4e15-8e1e-b912424a35b2": { + "ef4e5445-487e-4a0c-ac01-063e8c199a84": { "dataType": "string", "isBucketed": true, "label": "Top 1000 values of logstash.node.stats.logstash.name", @@ -183,10 +317,10 @@ "includeIsRegex": false, "missingBucket": false, "orderBy": { - "columnId": "a98fb979-a039-4322-871f-1d32480fd158", - "type": "column" + "fallback": true, + "type": "alphabetical" }, - "orderDirection": "desc", + "orderDirection": "asc", "otherBucket": true, "parentFormat": { "id": "terms" @@ -197,7 +331,8 @@ "sourceField": "logstash.node.stats.logstash.name" } }, - "incompleteColumns": {} + "incompleteColumns": {}, + "sampling": 1 } } }, @@ -212,139 +347,80 @@ "query": "" }, "visualization": { - "breakdownByAccessor": "b1f83d95-ed15-4e15-8e1e-b912424a35b2", - "collapseFn": "sum", - "layerId": "d944a0ae-9030-413e-afd6-c4a57ff34ad0", - "layerType": "data", - "metricAccessor": "a98fb979-a039-4322-871f-1d32480fd158", - "showBar": false - } - }, - "title": "Total JVM Heap Usage", - "type": "lens", - "visualizationType": "lnsMetric" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 4, - "i": "e4763613-7bc1-417e-8173-802c6342173b", - "w": 10, - "x": 18, - "y": 0 - }, - "panelIndex": "e4763613-7bc1-417e-8173-802c6342173b", - "title": "Total JVM Heap Usage", - "type": "lens", - "version": "8.7.1" - }, - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "bfb76152-1d30-4689-87e3-cf3b5803fdbf": { - "columnOrder": [ - "8ab113dc-30a3-4109-9b26-9e436fcb4291", - "9329ee1e-ade0-412e-86f4-9e901db45ab2" - ], - "columns": { - "8ab113dc-30a3-4109-9b26-9e436fcb4291": { - "customLabel": true, - "dataType": "string", - "isBucketed": true, - "label": "Total Events Received", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "9329ee1e-ade0-412e-86f4-9e901db45ab2", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 1000 - }, - "scale": "ordinal", - "sourceField": "logstash.node.stats.logstash.name" - }, - "9329ee1e-ade0-412e-86f4-9e901db45ab2": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.events.in: *" - }, - "isBucketed": false, - "label": "Events Received", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.events.in" - } - }, - "incompleteColumns": {}, - "sampling": 1 - } + "axisTitlesVisibilitySettings": { + "x": false, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "hideEndzones": true, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1" + ], + "collapseFn": "sum", + "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" + } + ] } + ], + "legend": { + "isVisible": true, + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true }, - "textBased": { - "layers": {} + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" } - }, - "filters": [], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "breakdownByAccessor": "8ab113dc-30a3-4109-9b26-9e436fcb4291", - "collapseFn": "sum", - "layerId": "bfb76152-1d30-4689-87e3-cf3b5803fdbf", - "layerType": "data", - "maxCols": 1000, - "metricAccessor": "9329ee1e-ade0-412e-86f4-9e901db45ab2" } }, - "title": "Total Events Received", + "title": "", "type": "lens", - "visualizationType": "lnsMetric" + "visualizationType": "lnsXY" }, "enhancements": {}, "hidePanelTitles": false }, "gridData": { - "h": 4, - "i": "eed5915d-7416-4c4d-94a1-c3564e7371b6", - "w": 10, + "h": 10, + "i": "3df9851c-7ac3-4bed-ade1-7e3ee0509971", + "w": 20, "x": 28, - "y": 0 + "y": 4 }, - "panelIndex": "eed5915d-7416-4c4d-94a1-c3564e7371b6", - "title": "Total Events Received", + "panelIndex": "3df9851c-7ac3-4bed-ade1-7e3ee0509971", + "title": "Events Received Rate/s", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -366,25 +442,111 @@ "ef4e5445-487e-4a0c-ac01-063e8c199a84", "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", "0c423bdf-578e-4f0a-bb27-180a24a133e1", - "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" ], "columns": { "0c423bdf-578e-4f0a-bb27-180a24a133e1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Event Latency (ms)", + "operationType": "formula", + "params": { + "formula": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", + "isFormulaBroken": false + }, + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" + ], + "scale": "ratio" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.events.in : *" + "query": "logstash.node.stats.events.duration_in_millis: *" }, "isBucketed": false, - "label": "Events Emitted Rate", + "label": "Part of Event Latency (ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.duration_in_millis" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", "operationType": "counter_rate", "references": [ - "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.events.out: *" + }, + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.out" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "counter_rate", + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2" ], "scale": "ratio", "timeScale": "s" }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" + ], + "location": { + "max": 128, + "min": 0 + }, + "name": "divide", + "text": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", + "type": "function" + } + }, + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" + ], + "scale": "ratio" + }, "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { "dataType": "date", "isBucketed": true, @@ -398,17 +560,6 @@ "scale": "interval", "sourceField": "@timestamp" }, - "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { - "dataType": "number", - "isBucketed": false, - "label": "Maximum of logstash.node.stats.events.out", - "operationType": "max", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.events.out" - }, "ef4e5445-487e-4a0c-ac01-063e8c199a84": { "dataType": "string", "isBucketed": true, @@ -473,7 +624,7 @@ "accessors": [ "0c423bdf-578e-4f0a-bb27-180a24a133e1" ], - "collapseFn": "sum", + "collapseFn": "avg", "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "layerType": "data", "position": "top", @@ -516,15 +667,15 @@ }, "gridData": { "h": 10, - "i": "a66223a5-9fdb-4335-8012-4ae2748928ac", + "i": "272e809f-0867-4ef2-aef3-626e954008c9", "w": 20, "x": 8, - "y": 4 + "y": 14 }, - "panelIndex": "a66223a5-9fdb-4335-8012-4ae2748928ac", - "title": "Events Emitted Rate/s", + "panelIndex": "272e809f-0867-4ef2-aef3-626e954008c9", + "title": "Events Latency (ms) average", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -532,7 +683,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "name": "indexpattern-datasource-layer-3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", "type": "index-pattern" } ], @@ -541,57 +692,25 @@ "datasourceStates": { "formBased": { "layers": { - "bfb76152-1d30-4689-87e3-cf3b5803fdbf": { + "3eb7cb2b-67b2-4fc7-b311-9106826dbbfc": { "columnOrder": [ - "8ab113dc-30a3-4109-9b26-9e436fcb4291", - "9329ee1e-ade0-412e-86f4-9e901db45ab2" + "a45f2761-e598-4ec1-8dff-95b395d0dd1e" ], "columns": { - "8ab113dc-30a3-4109-9b26-9e436fcb4291": { + "a45f2761-e598-4ec1-8dff-95b395d0dd1e": { "customLabel": true, - "dataType": "string", - "isBucketed": true, - "label": "Total Events Received", - "operationType": "terms", + "dataType": "number", + "isBucketed": false, + "label": " ", + "operationType": "unique_count", "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "9329ee1e-ade0-412e-86f4-9e901db45ab2", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 1000 - }, - "scale": "ordinal", - "sourceField": "logstash.node.stats.logstash.name" - }, - "9329ee1e-ade0-412e-86f4-9e901db45ab2": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.events.out: *" - }, - "isBucketed": false, - "label": "Events Emitted", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" + "emptyAsNull": true }, "scale": "ratio", - "sourceField": "logstash.node.stats.events.out" + "sourceField": "logstash.node.stats.logstash.name" } }, - "incompleteColumns": {}, - "sampling": 1 + "incompleteColumns": {} } } }, @@ -606,32 +725,27 @@ "query": "" }, "visualization": { - "breakdownByAccessor": "8ab113dc-30a3-4109-9b26-9e436fcb4291", - "collapseFn": "sum", - "layerId": "bfb76152-1d30-4689-87e3-cf3b5803fdbf", - "layerType": "data", - "maxCols": 1000, - "metricAccessor": "9329ee1e-ade0-412e-86f4-9e901db45ab2" + "accessor": "a45f2761-e598-4ec1-8dff-95b395d0dd1e", + "layerId": "3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", + "layerType": "data" } }, - "title": "Total Events Emitted", - "type": "lens", - "visualizationType": "lnsMetric" + "visualizationType": "lnsLegacyMetric" }, "enhancements": {}, "hidePanelTitles": false }, "gridData": { "h": 4, - "i": "d7c45290-1873-4a8d-9c12-4d009925d447", + "i": "0c5d0dc4-28b0-4875-a024-63fb45db1c37", "w": 10, - "x": 38, + "x": 8, "y": 0 }, - "panelIndex": "d7c45290-1873-4a8d-9c12-4d009925d447", - "title": "Total Events Emitted", + "panelIndex": "0c5d0dc4-28b0-4875-a024-63fb45db1c37", + "title": "Node Count", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -639,7 +753,12 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "name": "indexpattern-datasource-layer-d944a0ae-9030-413e-afd6-c4a57ff34ad0", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-1e302797-cf91-47b1-8b75-d9dacdffbd4a", "type": "index-pattern" } ], @@ -648,55 +767,109 @@ "datasourceStates": { "formBased": { "layers": { - "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { + "1e302797-cf91-47b1-8b75-d9dacdffbd4a": { "columnOrder": [ - "ef4e5445-487e-4a0c-ac01-063e8c199a84", - "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", - "0c423bdf-578e-4f0a-bb27-180a24a133e1", - "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + "e3571a52-74be-42e1-88a0-d3546248af15", + "7c5c2aaa-f43c-4e36-95d2-5f92e765c49f", + "e98a18a1-4df5-419f-a622-8286fc932f2a" ], "columns": { - "0c423bdf-578e-4f0a-bb27-180a24a133e1": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.events.in : *" - }, - "isBucketed": false, - "label": "Events Received Rate", - "operationType": "counter_rate", - "references": [ - "cd7a8cba-084f-42b4-a4b5-334eee79e32e" - ], - "scale": "ratio", - "timeScale": "s" - }, - "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "7c5c2aaa-f43c-4e36-95d2-5f92e765c49f": { "dataType": "date", "isBucketed": true, "label": "@timestamp", "operationType": "date_histogram", "params": { - "dropPartials": true, + "dropPartials": false, "includeEmptyRows": true, "interval": "auto" }, "scale": "interval", "sourceField": "@timestamp" }, - "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { + "e3571a52-74be-42e1-88a0-d3546248af15": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.node.stats.logstash.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "e98a18a1-4df5-419f-a622-8286fc932f2a", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.stats.logstash.name" + }, + "e98a18a1-4df5-419f-a622-8286fc932f2a": { + "customLabel": true, "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.jvm.mem.heap_used_in_bytes: *" + }, "isBucketed": false, - "label": "Maximum of logstash.node.stats.events.in", - "operationType": "max", + "label": "Heap Used", + "operationType": "last_value", "params": { - "emptyAsNull": true + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.events.in" + "sourceField": "logstash.node.stats.jvm.mem.heap_used_in_bytes" + } + }, + "incompleteColumns": {}, + "linkToLayers": [ + "d944a0ae-9030-413e-afd6-c4a57ff34ad0" + ], + "sampling": 1 + }, + "d944a0ae-9030-413e-afd6-c4a57ff34ad0": { + "columnOrder": [ + "b1f83d95-ed15-4e15-8e1e-b912424a35b2", + "a98fb979-a039-4322-871f-1d32480fd158" + ], + "columns": { + "a98fb979-a039-4322-871f-1d32480fd158": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.jvm.mem.heap_used_in_bytes: *" + }, + "isBucketed": false, + "label": "Heap Used", + "operationType": "last_value", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.jvm.mem.heap_used_in_bytes" }, - "ef4e5445-487e-4a0c-ac01-063e8c199a84": { + "b1f83d95-ed15-4e15-8e1e-b912424a35b2": { "dataType": "string", "isBucketed": true, "label": "Top 1000 values of logstash.node.stats.logstash.name", @@ -708,10 +881,10 @@ "includeIsRegex": false, "missingBucket": false, "orderBy": { - "fallback": true, - "type": "alphabetical" + "columnId": "a98fb979-a039-4322-871f-1d32480fd158", + "type": "column" }, - "orderDirection": "asc", + "orderDirection": "desc", "otherBucket": true, "parentFormat": { "id": "terms" @@ -722,8 +895,7 @@ "sourceField": "logstash.node.stats.logstash.name" } }, - "incompleteColumns": {}, - "sampling": 1 + "incompleteColumns": {} } } }, @@ -738,80 +910,35 @@ "query": "" }, "visualization": { - "axisTitlesVisibilitySettings": { - "x": false, - "yLeft": true, - "yRight": true - }, - "fittingFunction": "None", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "hideEndzones": true, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1" - ], - "collapseFn": "sum", - "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", - "layerType": "data", - "position": "top", - "seriesType": "line", - "showGridlines": false, - "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", - "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", - "yConfig": [ - { - "color": "#ff0000", - "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" - } - ] - } - ], - "legend": { - "isVisible": true, - "position": "bottom", - "shouldTruncate": false, - "showSingleSeries": true - }, - "preferredSeriesType": "line", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "hide", - "yLeftExtent": { - "mode": "dataBounds" - } + "breakdownByAccessor": "b1f83d95-ed15-4e15-8e1e-b912424a35b2", + "collapseFn": "sum", + "layerId": "d944a0ae-9030-413e-afd6-c4a57ff34ad0", + "layerType": "data", + "metricAccessor": "a98fb979-a039-4322-871f-1d32480fd158", + "showBar": false, + "trendlineBreakdownByAccessor": "e3571a52-74be-42e1-88a0-d3546248af15", + "trendlineLayerId": "1e302797-cf91-47b1-8b75-d9dacdffbd4a", + "trendlineLayerType": "metricTrendline", + "trendlineMetricAccessor": "e98a18a1-4df5-419f-a622-8286fc932f2a", + "trendlineTimeAccessor": "7c5c2aaa-f43c-4e36-95d2-5f92e765c49f" } }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" + "visualizationType": "lnsMetric" }, "enhancements": {}, "hidePanelTitles": false }, "gridData": { - "h": 10, - "i": "3df9851c-7ac3-4bed-ade1-7e3ee0509971", - "w": 20, - "x": 28, - "y": 4 + "h": 4, + "i": "c38da400-a6af-4225-9e21-6ba4da521b43", + "w": 10, + "x": 18, + "y": 0 }, - "panelIndex": "3df9851c-7ac3-4bed-ade1-7e3ee0509971", - "title": "Events Received Rate/s", + "panelIndex": "c38da400-a6af-4225-9e21-6ba4da521b43", + "title": "Total JVM Heap Used", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -819,7 +946,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "name": "indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", "type": "index-pattern" } ], @@ -828,133 +955,122 @@ "datasourceStates": { "formBased": { "layers": { - "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { + "bfb76152-1d30-4689-87e3-cf3b5803fdbf": { "columnOrder": [ - "ef4e5445-487e-4a0c-ac01-063e8c199a84", - "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", - "0c423bdf-578e-4f0a-bb27-180a24a133e1", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X0", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X2", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X3", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" + "8ab113dc-30a3-4109-9b26-9e436fcb4291", + "9329ee1e-ade0-412e-86f4-9e901db45ab2" ], "columns": { - "0c423bdf-578e-4f0a-bb27-180a24a133e1": { + "8ab113dc-30a3-4109-9b26-9e436fcb4291": { "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Event Latency (ms)", - "operationType": "formula", - "params": { - "formula": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", - "isFormulaBroken": false - }, - "references": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" - ], - "scale": "ratio" - }, - "0c423bdf-578e-4f0a-bb27-180a24a133e1X0": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.events.duration_in_millis: *" - }, - "isBucketed": false, - "label": "Part of Event Latency (ms)", - "operationType": "last_value", + "dataType": "string", + "isBucketed": true, + "label": "Total Events Received", + "operationType": "terms", "params": { - "sortField": "@timestamp" + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "9329ee1e-ade0-412e-86f4-9e901db45ab2", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 }, - "scale": "ratio", - "sourceField": "logstash.node.stats.events.duration_in_millis" - }, - "0c423bdf-578e-4f0a-bb27-180a24a133e1X1": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Event Latency (ms)", - "operationType": "counter_rate", - "references": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1X0" - ], - "scale": "ratio", - "timeScale": "s" + "scale": "ordinal", + "sourceField": "logstash.node.stats.logstash.name" }, - "0c423bdf-578e-4f0a-bb27-180a24a133e1X2": { + "9329ee1e-ade0-412e-86f4-9e901db45ab2": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.events.out: *" + "query": "logstash.node.stats.events.in: *" }, "isBucketed": false, - "label": "Part of Event Latency (ms)", + "label": "Events Received", "operationType": "last_value", "params": { "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.events.out" - }, - "0c423bdf-578e-4f0a-bb27-180a24a133e1X3": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Event Latency (ms)", - "operationType": "counter_rate", - "references": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1X2" - ], - "scale": "ratio", - "timeScale": "s" - }, - "0c423bdf-578e-4f0a-bb27-180a24a133e1X4": { + "sourceField": "logstash.node.stats.events.in" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "8ab113dc-30a3-4109-9b26-9e436fcb4291", + "collapseFn": "sum", + "layerId": "bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "layerType": "data", + "maxCols": 1000, + "metricAccessor": "9329ee1e-ade0-412e-86f4-9e901db45ab2" + } + }, + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 4, + "i": "944396f6-413e-439e-9226-5fcf76247442", + "w": 10, + "x": 28, + "y": 0 + }, + "panelIndex": "944396f6-413e-439e-9226-5fcf76247442", + "title": "Total Events Received", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "bfb76152-1d30-4689-87e3-cf3b5803fdbf": { + "columnOrder": [ + "8ab113dc-30a3-4109-9b26-9e436fcb4291", + "9329ee1e-ade0-412e-86f4-9e901db45ab2" + ], + "columns": { + "8ab113dc-30a3-4109-9b26-9e436fcb4291": { "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Event Latency (ms)", - "operationType": "math", - "params": { - "tinymathAst": { - "args": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" - ], - "location": { - "max": 128, - "min": 0 - }, - "name": "divide", - "text": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", - "type": "function" - } - }, - "references": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", - "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" - ], - "scale": "ratio" - }, - "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { - "dataType": "date", - "isBucketed": true, - "label": "@timestamp", - "operationType": "date_histogram", - "params": { - "dropPartials": true, - "includeEmptyRows": true, - "interval": "auto" - }, - "scale": "interval", - "sourceField": "@timestamp" - }, - "ef4e5445-487e-4a0c-ac01-063e8c199a84": { "dataType": "string", "isBucketed": true, - "label": "Top 1000 values of logstash.node.stats.logstash.name", + "label": "Total Events Received", "operationType": "terms", "params": { "exclude": [], @@ -963,10 +1079,10 @@ "includeIsRegex": false, "missingBucket": false, "orderBy": { - "fallback": true, - "type": "alphabetical" + "columnId": "9329ee1e-ade0-412e-86f4-9e901db45ab2", + "type": "column" }, - "orderDirection": "asc", + "orderDirection": "desc", "otherBucket": true, "parentFormat": { "id": "terms" @@ -975,6 +1091,22 @@ }, "scale": "ordinal", "sourceField": "logstash.node.stats.logstash.name" + }, + "9329ee1e-ade0-412e-86f4-9e901db45ab2": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.events.out: *" + }, + "isBucketed": false, + "label": "Events Emitted", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.out" } }, "incompleteColumns": {}, @@ -993,128 +1125,92 @@ "query": "" }, "visualization": { - "axisTitlesVisibilitySettings": { - "x": false, - "yLeft": true, - "yRight": true - }, - "fittingFunction": "None", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "hideEndzones": true, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "0c423bdf-578e-4f0a-bb27-180a24a133e1" - ], - "collapseFn": "avg", - "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", - "layerType": "data", - "position": "top", - "seriesType": "line", - "showGridlines": false, - "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", - "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", - "yConfig": [ - { - "color": "#ff0000", - "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" - } - ] - } - ], - "legend": { - "isVisible": true, - "position": "bottom", - "shouldTruncate": false, - "showSingleSeries": true - }, - "preferredSeriesType": "line", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "hide", - "yLeftExtent": { - "mode": "dataBounds" - } + "breakdownByAccessor": "8ab113dc-30a3-4109-9b26-9e436fcb4291", + "collapseFn": "sum", + "layerId": "bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "layerType": "data", + "maxCols": 1000, + "metricAccessor": "9329ee1e-ade0-412e-86f4-9e901db45ab2" } }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" + "visualizationType": "lnsMetric" }, "enhancements": {}, "hidePanelTitles": false }, "gridData": { - "h": 10, - "i": "272e809f-0867-4ef2-aef3-626e954008c9", - "w": 20, - "x": 8, - "y": 14 + "h": 4, + "i": "22f8be07-6626-4fc6-a741-e095030bf543", + "w": 10, + "x": 38, + "y": 0 }, - "panelIndex": "272e809f-0867-4ef2-aef3-626e954008c9", - "title": "Events Latency (ms) average", + "panelIndex": "22f8be07-6626-4fc6-a741-e095030bf543", + "title": "Total Events Emitted", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" } ], "timeRestore": false, "title": "[Metrics Logstash] Logstash Overview", "version": 1 }, - "coreMigrationVersion": "8.7.1", - "created_at": "2023-09-08T17:27:49.263Z", + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T14:11:03.240Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", - "migrationVersion": { - "dashboard": "8.7.0" - }, + "managed": true, "references": [ { "id": "logstash-sm-metrics", - "name": "e7bf7bee-710a-456a-995c-a570ee24cc05:indexpattern-datasource-layer-3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", + "name": "a66223a5-9fdb-4335-8012-4ae2748928ac:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "e4763613-7bc1-417e-8173-802c6342173b:indexpattern-datasource-layer-d944a0ae-9030-413e-afd6-c4a57ff34ad0", + "name": "3df9851c-7ac3-4bed-ade1-7e3ee0509971:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "eed5915d-7416-4c4d-94a1-c3564e7371b6:indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "name": "272e809f-0867-4ef2-aef3-626e954008c9:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "a66223a5-9fdb-4335-8012-4ae2748928ac:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "name": "0c5d0dc4-28b0-4875-a024-63fb45db1c37:indexpattern-datasource-layer-3eb7cb2b-67b2-4fc7-b311-9106826dbbfc", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "d7c45290-1873-4a8d-9c12-4d009925d447:indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "name": "c38da400-a6af-4225-9e21-6ba4da521b43:indexpattern-datasource-layer-d944a0ae-9030-413e-afd6-c4a57ff34ad0", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "3df9851c-7ac3-4bed-ade1-7e3ee0509971:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "name": "c38da400-a6af-4225-9e21-6ba4da521b43:indexpattern-datasource-layer-1e302797-cf91-47b1-8b75-d9dacdffbd4a", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "272e809f-0867-4ef2-aef3-626e954008c9:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "name": "944396f6-413e-439e-9226-5fcf76247442:indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "22f8be07-6626-4fc6-a741-e095030bf543:indexpattern-datasource-layer-bfb76152-1d30-4689-87e3-cf3b5803fdbf", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_5c0bd4d8-47ca-4a2f-a1ff-d9f4b55dfad0:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_c5cca387-507e-44a6-883f-0ab948c24319:optionsListDataView", "type": "index-pattern" } ], - "type": "dashboard" + "type": "dashboard", + "typeMigrationVersion": "8.9.0" } \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index 86edfd102f5..6c04491d3cc 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -4,7 +4,7 @@ "chainingSystem": "HIERARCHICAL", "controlStyle": "oneLine", "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"f8b74dd1-8785-4fff-96ba-8d8cfb7615eb\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"f8b74dd1-8785-4fff-96ba-8d8cfb7615eb\",\"fieldName\":\"logstash.node.stats.logstash.name\",\"title\":\"Node Name\",\"enhancements\":{}}}}" + "panelsJSON": "{\"f8e0eae3-62e0-4539-87d6-523e89a8213e\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"f8e0eae3-62e0-4539-87d6-523e89a8213e\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"25b6ddfd-e372-4344-827b-c81461088031\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"25b6ddfd-e372-4344-827b-c81461088031\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" }, "description": "", "kibanaSavedObjectMeta": { @@ -43,7 +43,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n**[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d)** \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\nOverview\n\nThis Dashboard gives a view of the overall performance of a single Logstash Node. Should be used in conjunction with the node name\nfilter", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n**[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d)** \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\nOverview\n\nThis Dashboard gives a view of the overall performance of a single Logstash Node. Should be used in conjunction with the node name\nfilter", "openLinksInNewTab": false }, "title": "", @@ -52,7 +52,7 @@ } }, "gridData": { - "h": 31, + "h": 38, "i": "c2c433cf-50ce-4530-86e5-f82a240c57b8", "w": 8, "x": 0, @@ -60,11 +60,12 @@ }, "panelIndex": "c2c433cf-50ce-4530-86e5-f82a240c57b8", "type": "visualization", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -99,7 +100,7 @@ "query": "logstash.node.stats.logstash.http_address: *" }, "isBucketed": false, - "label": "HTTP Address", + "label": "HTTP address", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -115,7 +116,7 @@ "query": "logstash.node.stats.events.out: *" }, "isBucketed": false, - "label": "Events Emitted", + "label": "Events emitted", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -150,7 +151,7 @@ "query": "logstash.node.stats.reloads.successes: *" }, "isBucketed": false, - "label": "Successful Config Reloads", + "label": "Successful config reloads", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -162,7 +163,7 @@ "customLabel": true, "dataType": "string", "isBucketed": true, - "label": "Node Name", + "label": "Logstash host name", "operationType": "terms", "params": { "exclude": [], @@ -189,7 +190,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Failed Config Reloads", + "label": "Failed config reloads", "operationType": "median", "params": { "emptyAsNull": true @@ -221,7 +222,7 @@ "query": "logstash.node.stats.logstash.pipeline.workers: *" }, "isBucketed": false, - "label": "Default Pipeline Workers", + "label": "Default pipeline workers", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -233,7 +234,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Default Batch size", + "label": "Default batch size", "operationType": "median", "params": { "emptyAsNull": true @@ -249,7 +250,7 @@ "query": "logstash.node.stats.events.in: *" }, "isBucketed": false, - "label": "Events Received", + "label": "Events received", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -347,7 +348,7 @@ "panelIndex": "73a755c6-89a3-4f34-8daf-83feef5caa28", "title": "", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -505,11 +506,12 @@ "panelIndex": "63747092-edb6-4864-a9ad-27e5bdce2ad2", "title": "Events Received Rate (/s)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -534,7 +536,7 @@ "dataType": "number", "filter": { "language": "kuery", - "query": "logstash_stats.jvm.mem.heap_used_in_bytes: *" + "query": "\"logstash.node.stats.jvm.mem.heap_used_in_bytes\": *" }, "isBucketed": false, "label": "JVM Heap (MB)", @@ -549,7 +551,7 @@ "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash_stats.jvm.mem.heap_used_in_bytes" + "sourceField": "logstash.node.stats.jvm.mem.heap_used_in_bytes" }, "467daee5-8948-4576-80fc-12bb76b01c0f": { "dataType": "date", @@ -673,7 +675,7 @@ "panelIndex": "9ea7a32a-ee7e-45d4-b0cf-273278e52cae", "title": "JVM Heap (MB)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -831,7 +833,7 @@ "panelIndex": "bac81244-9c35-4cf9-8ed4-3c7082a255ae", "title": "Events Emitted Rate (/s)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -971,7 +973,7 @@ "panelIndex": "5a43f153-bec9-4420-96f8-0c2d4b032a43", "title": "Process CPU Utilization (%)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -1203,7 +1205,7 @@ "panelIndex": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", "title": "Events Latency (ms)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -1343,19 +1345,233 @@ "panelIndex": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", "title": "System Load", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-2fb7e546-2fca-4965-a4d3-dcc063be3ded", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "2fb7e546-2fca-4965-a4d3-dcc063be3ded": { + "columnOrder": [ + "2d3cd5ec-a0d0-4569-8c05-ea9f847a3623", + "ca4b5da2-8007-4f89-8261-9081ee13cdd5", + "0c0ae432-64ed-4792-b539-c195c3d96549" + ], + "columns": { + "0c0ae432-64ed-4792-b539-c195c3d96549": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.out: *" + }, + "isBucketed": false, + "label": "Total events received", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "2d3cd5ec-a0d0-4569-8c05-ea9f847a3623": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "0c0ae432-64ed-4792-b539-c195c3d96549", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + }, + "ca4b5da2-8007-4f89-8261-9081ee13cdd5": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Total events received", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "columnId": "2d3cd5ec-a0d0-4569-8c05-ea9f847a3623", + "isTransposed": false + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "0c0ae432-64ed-4792-b539-c195c3d96549", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "ca4b5da2-8007-4f89-8261-9081ee13cdd5", + "isTransposed": false, + "palette": { + "name": "positive", + "params": { + "stops": [ + { + "color": "#d6e9e4", + "stop": 20 + }, + { + "color": "#aed3ca", + "stop": 40 + }, + { + "color": "#85bdb1", + "stop": 60 + }, + { + "color": "#5aa898", + "stop": 80 + }, + { + "color": "#209280", + "stop": 100 + } + ] + }, + "type": "palette" + } + } + ], + "layerId": "2fb7e546-2fca-4965-a4d3-dcc063be3ded", + "layerType": "data" + } + }, + "title": "Running Pipelines", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "Go to Dashboard" + }, + "eventId": "32da855b-5a16-4271-8e06-938568d20659", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "87bc08bb-a0ca-4f82-8943-a141aaef3248", + "w": 40, + "x": 8, + "y": 31 + }, + "panelIndex": "87bc08bb-a0ca-4f82-8943-a141aaef3248", + "title": "Running Pipelines", + "type": "lens", + "version": "8.10.1" } ], "timeRestore": false, "title": "[Metrics Logstash] Single Node Overview", "version": 1 }, - "coreMigrationVersion": "8.7.1", - "created_at": "2023-09-08T17:43:03.843Z", + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T19:56:39.887Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", - "migrationVersion": { - "dashboard": "8.7.0" - }, + "managed": false, "references": [ { "id": "logstash-sm-metrics", @@ -1394,9 +1610,25 @@ }, { "id": "logstash-sm-metrics", - "name": "controlGroup_f8b74dd1-8785-4fff-96ba-8d8cfb7615eb:optionsListDataView", + "name": "87bc08bb-a0ca-4f82-8943-a141aaef3248:indexpattern-datasource-layer-2fb7e546-2fca-4965-a4d3-dcc063be3ded", + "type": "index-pattern" + }, + { + "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", + "name": "87bc08bb-a0ca-4f82-8943-a141aaef3248:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:32da855b-5a16-4271-8e06-938568d20659:dashboardId", + "type": "dashboard" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_f8e0eae3-62e0-4539-87d6-523e89a8213e:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_25b6ddfd-e372-4344-827b-c81461088031:optionsListDataView", "type": "index-pattern" } ], - "type": "dashboard" + "type": "dashboard", + "typeMigrationVersion": "8.9.0" } \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index 9b0f6b91bba..2490aa9c8b3 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -4,7 +4,7 @@ "chainingSystem": "HIERARCHICAL", "controlStyle": "oneLine", "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"4b095518-7dc3-4d0e-a323-5d7846f65bff\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"4b095518-7dc3-4d0e-a323-5d7846f65bff\",\"fieldName\":\"logstash.node.stats.logstash.name\",\"title\":\"Node Name\",\"enhancements\":{}}}}" + "panelsJSON": "{\"6e3b65dd-ff37-486c-a838-8355772fb76d\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"6e3b65dd-ff37-486c-a838-8355772fb76d\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"939358a9-ac4d-4219-9c2f-7e39d482868c\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"939358a9-ac4d-4219-9c2f-7e39d482868c\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" }, "description": "", "kibanaSavedObjectMeta": { @@ -42,7 +42,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n**[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5)** \n\nOverview\n\nThis Dashboard gives an advanced view of the overall performance of a single Logstash Node. Should be used in conjunction with the node name\nfilter", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n**[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5)** \n\n[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\nOverview\n\nThis Dashboard gives an advanced view of the overall performance of a single Logstash Node. Should be used in conjunction with the node name\nfilter", "openLinksInNewTab": false }, "title": "", @@ -51,7 +51,7 @@ } }, "gridData": { - "h": 34, + "h": 44, "i": "231de1ff-a22e-495c-9c97-20b20592ac46", "w": 8, "x": 0, @@ -59,11 +59,12 @@ }, "panelIndex": "231de1ff-a22e-495c-9c97-20b20592ac46", "type": "visualization", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -182,7 +183,7 @@ "size": 1000 }, "scale": "ordinal", - "sourceField": "logstash.node.stats.logstash.name" + "sourceField": "logstash.host.name" }, "a51eda03-44a2-4173-be51-72948417a027": { "customLabel": true, @@ -334,7 +335,7 @@ "visualizationType": "lnsDatatable" }, "enhancements": {}, - "hidePanelTitles": false + "hidePanelTitles": true }, "gridData": { "h": 4, @@ -344,13 +345,14 @@ "y": 0 }, "panelIndex": "25758935-54d2-4161-a037-800740bee913", - "title": "[Metrics Logstash] Node Summary Table", + "title": "Node Summary Table", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -366,9 +368,26 @@ "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ "467daee5-8948-4576-80fc-12bb76b01c0f", - "8b34584c-a413-4629-898d-cda409db5fe0" + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" ], "columns": { + "38a33c26-755e-4cf2-a5f2-739bf038af97": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.jvm.threads.count\": *" + }, + "isBucketed": false, + "label": "Threads in use", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.jvm.threads.count" + }, "467daee5-8948-4576-80fc-12bb76b01c0f": { "dataType": "date", "isBucketed": true, @@ -382,27 +401,21 @@ "scale": "interval", "sourceField": "@timestamp" }, - "8b34584c-a413-4629-898d-cda409db5fe0": { + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cpu.percent: *" + "query": "\"logstash.node.stats.jvm.threads.peak_count\": *" }, "isBucketed": false, - "label": "OS CPU Utilization (%)", + "label": "Peak threads in use", "operationType": "last_value", "params": { - "format": { - "id": "number", - "params": { - "decimals": 2 - } - }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cpu.percent" + "sourceField": "logstash.node.stats.jvm.threads.peak_count" } }, "incompleteColumns": {}, @@ -440,7 +453,8 @@ "layers": [ { "accessors": [ - "8b34584c-a413-4629-898d-cda409db5fe0" + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -449,16 +463,20 @@ "showGridlines": false, "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", "yConfig": [ + { + "color": "#3078c0", + "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" + }, { "color": "#ff0000", - "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" + "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" } ] } ], "legend": { "isVisible": true, - "position": "right" + "position": "bottom" }, "preferredSeriesType": "line", "tickLabelsVisibilitySettings": { @@ -478,15 +496,15 @@ }, "gridData": { "h": 10, - "i": "5a43f153-bec9-4420-96f8-0c2d4b032a43", + "i": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", "w": 20, "x": 8, "y": 4 }, - "panelIndex": "5a43f153-bec9-4420-96f8-0c2d4b032a43", - "title": "OS CPU Utilization (%)", + "panelIndex": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", + "title": "Number of Java Threads", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -626,15 +644,16 @@ "panelIndex": "494716ff-128d-4a72-bf41-2f6f42dd5158", "title": "Process CPU Utilization (%)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "indexpattern-datasource-layer-10c3cc35-e7f6-4a97-a655-ce7b79e6a395", "type": "index-pattern" } ], @@ -643,30 +662,17 @@ "datasourceStates": { "formBased": { "layers": { - "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { + "10c3cc35-e7f6-4a97-a655-ce7b79e6a395": { "columnOrder": [ - "467daee5-8948-4576-80fc-12bb76b01c0f", - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "f2b94321-1b01-4dd3-88d9-cbb455fa0b7f", + "6ef4f800-7f38-4a1a-adc4-737d68f70496", + "b87af172-05ed-43d1-aaf0-191042e36cf3", + "b87af172-05ed-43d1-aaf0-191042e36cf3X0", + "b87af172-05ed-43d1-aaf0-191042e36cf3X1", + "b87af172-05ed-43d1-aaf0-191042e36cf3X2" ], "columns": { - "38a33c26-755e-4cf2-a5f2-739bf038af97": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.process.open_file_descriptors: *" - }, - "isBucketed": false, - "label": "File Descriptors in use", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.process.open_file_descriptors" - }, - "467daee5-8948-4576-80fc-12bb76b01c0f": { + "6ef4f800-7f38-4a1a-adc4-737d68f70496": { "dataType": "date", "isBucketed": true, "label": "@timestamp", @@ -679,21 +685,122 @@ "scale": "interval", "sourceField": "@timestamp" }, - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "b87af172-05ed-43d1-aaf0-191042e36cf3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Persistent Queue Utilization", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "round(last_value(logstash.pipeline.total.queues.current_size.bytes, kql='logstash.pipeline.total.queues.current_size.bytes: *')/last_value(logstash.pipeline.total.queues.max_size.bytes, kql='logstash.pipeline.total.queues.max_size.bytes: *'), 2)", + "isFormulaBroken": false + }, + "references": [ + "b87af172-05ed-43d1-aaf0-191042e36cf3X2" + ], + "scale": "ratio" + }, + "b87af172-05ed-43d1-aaf0-191042e36cf3X0": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.process.max_file_descriptors: *" + "query": "logstash.pipeline.total.queues.current_size.bytes: *" }, "isBucketed": false, - "label": "Max File Descriptors", + "label": "Part of Persistent Queue Utilization", "operationType": "last_value", "params": { "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.process.max_file_descriptors" + "sourceField": "logstash.pipeline.total.queues.current_size.bytes" + }, + "b87af172-05ed-43d1-aaf0-191042e36cf3X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.queues.max_size.bytes: *" + }, + "isBucketed": false, + "label": "Part of Persistent Queue Utilization", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.max_size.bytes" + }, + "b87af172-05ed-43d1-aaf0-191042e36cf3X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Persistent Queue Utilization", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + { + "args": [ + "b87af172-05ed-43d1-aaf0-191042e36cf3X0", + "b87af172-05ed-43d1-aaf0-191042e36cf3X1" + ], + "location": { + "max": 241, + "min": 6 + }, + "name": "divide", + "text": "last_value(logstash.pipeline.total.queues.current_size.bytes, kql='logstash.pipeline.total.queues.current_size.bytes: *')/last_value(logstash.pipeline.total.queues.max_size.bytes, kql='logstash.pipeline.total.queues.max_size.bytes: *')", + "type": "function" + }, + 2 + ], + "location": { + "max": 245, + "min": 0 + }, + "name": "round", + "text": "round(last_value(logstash.pipeline.total.queues.current_size.bytes, kql='logstash.pipeline.total.queues.current_size.bytes: *')/last_value(logstash.pipeline.total.queues.max_size.bytes, kql='logstash.pipeline.total.queues.max_size.bytes: *'), 2)", + "type": "function" + } + }, + "references": [ + "b87af172-05ed-43d1-aaf0-191042e36cf3X0", + "b87af172-05ed-43d1-aaf0-191042e36cf3X1" + ], + "scale": "ratio" + }, + "f2b94321-1b01-4dd3-88d9-cbb455fa0b7f": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" } }, "incompleteColumns": {}, @@ -731,38 +838,33 @@ "layers": [ { "accessors": [ - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "b87af172-05ed-43d1-aaf0-191042e36cf3" ], - "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "layerId": "10c3cc35-e7f6-4a97-a655-ce7b79e6a395", "layerType": "data", "position": "top", - "seriesType": "line", + "seriesType": "area", "showGridlines": false, - "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", - "yConfig": [ - { - "color": "#3078c0", - "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" - }, - { - "color": "#ff0000", - "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" - } - ] + "splitAccessor": "f2b94321-1b01-4dd3-88d9-cbb455fa0b7f", + "xAccessor": "6ef4f800-7f38-4a1a-adc4-737d68f70496" } ], "legend": { "isVisible": true, "position": "bottom" }, - "preferredSeriesType": "line", + "preferredSeriesType": "area", "tickLabelsVisibilitySettings": { "x": true, "yLeft": true, "yRight": true }, - "valueLabels": "hide" + "valueLabels": "hide", + "yLeftExtent": { + "lowerBound": 0, + "mode": "custom", + "upperBound": 1 + } } }, "title": "", @@ -774,15 +876,15 @@ }, "gridData": { "h": 10, - "i": "92979e91-3f35-4715-b97a-f088bb2eae51", + "i": "5a8acef6-7a36-4854-b557-69db174670b4", "w": 20, "x": 8, "y": 14 }, - "panelIndex": "92979e91-3f35-4715-b97a-f088bb2eae51", - "title": "Number of File Descriptors", + "panelIndex": "5a8acef6-7a36-4854-b557-69db174670b4", + "title": "Persistent Queue Utilization", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -801,17 +903,20 @@ "layers": { "336e8749-bfb1-4c0e-9f41-3892faebc198": { "columnOrder": [ + "95800073-8c87-4fee-8e70-b3ed0af3931b", "51d651dd-246b-40fc-aca2-f9e3cb50e23f", - "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b", - "7c151831-4457-4cec-a257-0e2fe2453e5a" + "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" ], "columns": { "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b": { - "customLabel": true, "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.queues.events: *" + }, "isBucketed": false, - "label": "Events Queued", - "operationType": "counter_rate", + "label": "Last value of logstash.pipeline.total.queues.events", + "operationType": "last_value", "params": { "format": { "id": "number", @@ -819,12 +924,11 @@ "decimals": 0, "suffix": "" } - } + }, + "sortField": "@timestamp" }, - "references": [ - "7c151831-4457-4cec-a257-0e2fe2453e5a" - ], - "scale": "ratio" + "scale": "ratio", + "sourceField": "logstash.pipeline.total.queues.events" }, "51d651dd-246b-40fc-aca2-f9e3cb50e23f": { "dataType": "date", @@ -839,16 +943,30 @@ "scale": "interval", "sourceField": "@timestamp" }, - "7c151831-4457-4cec-a257-0e2fe2453e5a": { - "dataType": "number", - "isBucketed": false, - "label": "Maximum of logstash.node.stats.queue.events_count", - "operationType": "max", + "95800073-8c87-4fee-8e70-b3ed0af3931b": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of logstash.pipeline.name", + "operationType": "terms", "params": { - "emptyAsNull": true + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 }, - "scale": "ratio", - "sourceField": "logstash.node.stats.queue.events_count" + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" } }, "incompleteColumns": {}, @@ -893,6 +1011,7 @@ "position": "top", "seriesType": "line", "showGridlines": false, + "splitAccessor": "95800073-8c87-4fee-8e70-b3ed0af3931b", "xAccessor": "51d651dd-246b-40fc-aca2-f9e3cb50e23f", "yConfig": [ { @@ -936,7 +1055,7 @@ "panelIndex": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", "title": "Persistent Queue Events", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -956,8 +1075,7 @@ "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ "467daee5-8948-4576-80fc-12bb76b01c0f", - "8b34584c-a413-4629-898d-cda409db5fe0", - "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + "8b34584c-a413-4629-898d-cda409db5fe0" ], "columns": { "467daee5-8948-4576-80fc-12bb76b01c0f": { @@ -973,51 +1091,27 @@ "scale": "interval", "sourceField": "@timestamp" }, - "4c49f25e-d518-4d91-bcdc-69ea91729ca3": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos: *" - }, - "isBucketed": false, - "label": "CGroup Throttling (ns)", - "operationType": "last_value", - "params": { - "format": { - "id": "number", - "params": { - "decimals": 1, - "suffix": "ns" - } - }, - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos" - }, "8b34584c-a413-4629-898d-cda409db5fe0": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos: *" + "query": "logstash.node.stats.os.cpu.percent: *" }, "isBucketed": false, - "label": "CGroup CPU Performance (ns)", + "label": "OS CPU Utilization (%)", "operationType": "last_value", "params": { "format": { "id": "number", "params": { - "decimals": 1, - "suffix": "ns" + "decimals": 2 } }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos" + "sourceField": "logstash.node.stats.os.cpu.percent" } }, "incompleteColumns": {}, @@ -1055,8 +1149,7 @@ "layers": [ { "accessors": [ - "8b34584c-a413-4629-898d-cda409db5fe0", - "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + "8b34584c-a413-4629-898d-cda409db5fe0" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -1068,17 +1161,13 @@ { "color": "#ff0000", "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" - }, - { - "color": "#6092c0", - "forAccessor": "4c49f25e-d518-4d91-bcdc-69ea91729ca3" } ] } ], "legend": { "isVisible": true, - "position": "bottom" + "position": "right" }, "preferredSeriesType": "line", "tickLabelsVisibilitySettings": { @@ -1098,15 +1187,15 @@ }, "gridData": { "h": 10, - "i": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", + "i": "5a43f153-bec9-4420-96f8-0c2d4b032a43", "w": 20, "x": 8, "y": 24 }, - "panelIndex": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", - "title": "CGroup Cpu Performance (ns)", + "panelIndex": "5a43f153-bec9-4420-96f8-0c2d4b032a43", + "title": "OS CPU Utilization (%)", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -1276,38 +1365,362 @@ "panelIndex": "71538b79-3608-4191-b3eb-0f898b7d7a3b", "title": "CGroup CFS Stats", "type": "lens", - "version": "8.7.1" - } - ], - "timeRestore": false, - "title": "[Metrics Logstash] Single Node Advanced View", - "version": 1 - }, - "coreMigrationVersion": "8.7.1", - "created_at": "2023-09-08T17:42:20.019Z", - "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", - "migrationVersion": { - "dashboard": "8.7.0" - }, - "references": [ - { - "id": "logstash-sm-metrics", - "name": "25758935-54d2-4161-a037-800740bee913:indexpattern-datasource-layer-f589d4f8-9213-4405-a5cd-72e93d3e2594", - "type": "index-pattern" - }, - { - "id": "logstash-sm-metrics", - "name": "5a43f153-bec9-4420-96f8-0c2d4b032a43:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", - "type": "index-pattern" - }, - { - "id": "logstash-sm-metrics", - "name": "494716ff-128d-4a72-bf41-2f6f42dd5158:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", - "type": "index-pattern" - }, - { + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { + "columnOrder": [ + "467daee5-8948-4576-80fc-12bb76b01c0f", + "8b34584c-a413-4629-898d-cda409db5fe0", + "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + ], + "columns": { + "467daee5-8948-4576-80fc-12bb76b01c0f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "4c49f25e-d518-4d91-bcdc-69ea91729ca3": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos: *" + }, + "isBucketed": false, + "label": "CGroup Throttling (ns)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": "ns" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos" + }, + "8b34584c-a413-4629-898d-cda409db5fe0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos: *" + }, + "isBucketed": false, + "label": "CGroup CPU Performance (ns)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": "ns" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "8b34584c-a413-4629-898d-cda409db5fe0", + "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + ], + "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" + }, + { + "color": "#6092c0", + "forAccessor": "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 10, + "i": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", + "w": 20, + "x": 28, + "y": 34 + }, + "panelIndex": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", + "title": "CGroup Cpu Performance (ns)", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { + "columnOrder": [ + "467daee5-8948-4576-80fc-12bb76b01c0f", + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + ], + "columns": { + "38a33c26-755e-4cf2-a5f2-739bf038af97": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.process.open_file_descriptors: *" + }, + "isBucketed": false, + "label": "File Descriptors in use", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.open_file_descriptors" + }, + "467daee5-8948-4576-80fc-12bb76b01c0f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.process.max_file_descriptors: *" + }, + "isBucketed": false, + "label": "Max File Descriptors", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.max_file_descriptors" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + ], + "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", + "yConfig": [ + { + "color": "#3078c0", + "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" + }, + { + "color": "#ff0000", + "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 10, + "i": "92979e91-3f35-4715-b97a-f088bb2eae51", + "w": 20, + "x": 8, + "y": 34 + }, + "panelIndex": "92979e91-3f35-4715-b97a-f088bb2eae51", + "title": "Number of File Descriptors", + "type": "lens", + "version": "8.10.1" + } + ], + "timeRestore": false, + "title": "[Metrics Logstash] Single Node Advanced View", + "version": 1 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T21:52:39.962Z", + "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", + "managed": false, + "references": [ + { "id": "logstash-sm-metrics", - "name": "92979e91-3f35-4715-b97a-f088bb2eae51:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "25758935-54d2-4161-a037-800740bee913:indexpattern-datasource-layer-f589d4f8-9213-4405-a5cd-72e93d3e2594", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "494716ff-128d-4a72-bf41-2f6f42dd5158:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "5a8acef6-7a36-4854-b557-69db174670b4:indexpattern-datasource-layer-10c3cc35-e7f6-4a97-a655-ce7b79e6a395", "type": "index-pattern" }, { @@ -1317,7 +1730,7 @@ }, { "id": "logstash-sm-metrics", - "name": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "5a43f153-bec9-4420-96f8-0c2d4b032a43:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { @@ -1327,9 +1740,25 @@ }, { "id": "logstash-sm-metrics", - "name": "controlGroup_4b095518-7dc3-4d0e-a323-5d7846f65bff:optionsListDataView", + "name": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "92979e91-3f35-4715-b97a-f088bb2eae51:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_6e3b65dd-ff37-486c-a838-8355772fb76d:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_939358a9-ac4d-4219-9c2f-7e39d482868c:optionsListDataView", "type": "index-pattern" } ], - "type": "dashboard" + "type": "dashboard", + "typeMigrationVersion": "8.9.0" } \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 189a8f21cea..32f568213ab 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -1,5 +1,11 @@ { "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"55797c81-115d-42f4-8b92-a90449de3183\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"55797c81-115d-42f4-8b92-a90449de3183\",\"fieldName\":\"logstash.host.name\",\"title\":\"logstash.host.name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"logstash.pipeline.name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" + }, "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { @@ -36,7 +42,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n**[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a)** \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\nOverview\n\nThis Dashboard gives an overall view of Logstash Nodes", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n**[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a)** \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\nOverview\n\nThis Dashboard gives an overall view of Logstash Nodes", "openLinksInNewTab": false }, "title": "", @@ -53,7 +59,7 @@ }, "panelIndex": "3175d525-4aa7-40b5-bc68-d89d105257de", "type": "visualization", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -123,9 +129,8 @@ "y": 0 }, "panelIndex": "8302492d-1d16-4955-91cd-c892d7002dbb", - "title": "Node Count", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -236,7 +241,7 @@ "panelIndex": "345f1c7e-4b91-4df2-8c09-22d2a8c5d6be", "title": "Total JVM Heap Used", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -343,7 +348,7 @@ "panelIndex": "b1f30ec6-50f8-4fb0-8ebf-c00b1df332ee", "title": "Total Events Received", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { @@ -450,11 +455,12 @@ "panelIndex": "5eedee54-06fd-496e-8b1d-b3df3ff80341", "title": "Total Events Emitted", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -635,7 +641,7 @@ "customLabel": true, "dataType": "string", "isBucketed": true, - "label": "Node Name", + "label": "Logstash Host Name", "operationType": "terms", "params": { "exclude": [], @@ -662,10 +668,11 @@ "parentFormat": { "id": "terms" }, + "secondaryFields": [], "size": 1000 }, "scale": "ordinal", - "sourceField": "logstash.node.stats.logstash.name" + "sourceField": "logstash.host.name" } }, "incompleteColumns": {}, @@ -882,7 +889,8 @@ } ] } - } + }, + "hidePanelTitles": true }, "gridData": { "h": 10, @@ -893,19 +901,17 @@ }, "panelIndex": "4a093412-9812-433a-bf8d-225e4a402339", "type": "lens", - "version": "8.7.1" + "version": "8.10.1" } ], "timeRestore": false, "title": "[Metrics Logstash] Nodes Overview", "version": 1 }, - "coreMigrationVersion": "8.7.1", - "created_at": "2023-09-08T17:27:49.263Z", + "coreMigrationVersion": "8.8.0", + "created_at": "2023-10-02T19:47:14.298Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", - "migrationVersion": { - "dashboard": "8.7.0" - }, + "managed": false, "references": [ { "id": "logstash-sm-metrics", @@ -936,7 +942,18 @@ "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", "name": "4a093412-9812-433a-bf8d-225e4a402339:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:1c4772e7-cf9a-417d-803f-3c7184b1ec82:dashboardId", "type": "dashboard" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_55797c81-115d-42f4-8b92-a90449de3183:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "controlGroup_ffddbdfd-c666-4a7c-b81d-c66cb20212f4:optionsListDataView", + "type": "index-pattern" } ], - "type": "dashboard" + "type": "dashboard", + "typeMigrationVersion": "8.9.0" } \ No newline at end of file From 780f6659a70d046176127e9672dce98c2c06bebb Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 3 Oct 2023 18:11:07 -0400 Subject: [PATCH 06/25] Tweaks and fixes Fix tests --- .../_dev/test/system/test-default-config.yml | 4 +- .../node_cel/agent/stream/stream.yml.hbs | 6 +- .../data_stream/node_cel/fields/fields.yml | 34 +- .../node_cel/fields/package-fields.yml | 2 + .../data_stream/node_cel/manifest.yml | 5 +- .../data_stream/node_cel/sample_event.json | 183 ++--- .../_dev/test/system/test-default-config.yml | 2 +- .../data_stream/pipeline/fields/fields.yml | 12 +- .../pipeline/fields/package-fields.yml | 2 + .../data_stream/pipeline/manifest.yml | 5 +- .../data_stream/pipeline/sample_event.json | 138 ++-- .../_dev/test/system/test-default-config.yml | 4 +- .../plugins/fields/package-fields.yml | 2 + .../logstash/data_stream/plugins/manifest.yml | 7 +- .../data_stream/plugins/sample_event.json | 130 ++-- ...-79270240-48ee-11ee-8cb5-99927777c522.json | 2 +- ...-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 7 +- ...-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 633 ++++++++++++------ ...-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 238 +++---- ...-c0594170-526a-11ee-9ecc-31444cb79548.json | 6 +- ...-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 4 +- 21 files changed, 807 insertions(+), 619 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml index 51f3a692227..be0cc2066ca 100644 --- a/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/node_cel/_dev/test/system/test-default-config.yml @@ -1,6 +1,6 @@ -type: logstash/metrics +type: cel dataset: logstash.node vars: - hosts: + url: - "http://{{Hostname}}:9600" data_stream: ~ diff --git a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs index 2f73ba745fb..11d5ffab915 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs @@ -36,7 +36,11 @@ program: | .as(body, {"b":body} .with({"logstash":{"node":{"stats":{ "events":body['events'], - "jvm":body['jvm'], + "jvm":{ + "uptime_in_millis":body['jvm']['uptime_in_millis'], + "mem":body['jvm']['mem'].drop("pools"), + "threads":body['jvm']['threads'] + }, "queue":body['queue'], "reloads":body['reloads'], "process":body['process'], diff --git a/packages/logstash/data_stream/node_cel/fields/fields.yml b/packages/logstash/data_stream/node_cel/fields/fields.yml index c0cf437ff00..dc9f266a74e 100644 --- a/packages/logstash/data_stream/node_cel/fields/fields.yml +++ b/packages/logstash/data_stream/node_cel/fields/fields.yml @@ -48,6 +48,12 @@ type: long - name: heap_used_percent type: long + - name: heap_committed_in_bytes + type: long + - name: non_heap_committed_in_bytes + type: long + - name: non_heap_used_in_bytes + type: long - name: gc type: group fields: @@ -85,6 +91,8 @@ Filtered events counter. - name: duration_in_millis type: long + - name: queue_push_duration_in_millis + type: long - name: logstash type: group fields: @@ -111,6 +119,8 @@ fields: - name: batch_size type: long + - name: batch_delay + type: long - name: workers type: long - name: os @@ -121,6 +131,8 @@ fields: - name: percent type: double + - name: total_in_millis + type: long - name: load_average type: group fields: @@ -159,12 +171,30 @@ - name: process type: group fields: - - name: cpu.percent - type: double + - name: cpu + type: group + fields: + - name: percent + type: double + - name: total_in_millis + type: long + - name: load_average + type: group + fields: + - name: 15m + type: half_float + - name: 1m + type: half_float + - name: 5m + type: half_float + - name: mem.total_virtual_in_bytes + type: long - name: max_file_descriptors type: long - name: open_file_descriptors type: long + - name: peak_open_file_descriptors + type: long - name: queue.events_count type: long - name: pipelines diff --git a/packages/logstash/data_stream/node_cel/fields/package-fields.yml b/packages/logstash/data_stream/node_cel/fields/package-fields.yml index 7464f183f19..5d270f86737 100644 --- a/packages/logstash/data_stream/node_cel/fields/package-fields.yml +++ b/packages/logstash/data_stream/node_cel/fields/package-fields.yml @@ -1,3 +1,5 @@ +- name: input.type + type: keyword - name: cluster_uuid type: alias path: logstash.elasticsearch.cluster.id diff --git a/packages/logstash/data_stream/node_cel/manifest.yml b/packages/logstash/data_stream/node_cel/manifest.yml index 71da9eee957..e2b87d773bc 100644 --- a/packages/logstash/data_stream/node_cel/manifest.yml +++ b/packages/logstash/data_stream/node_cel/manifest.yml @@ -1,6 +1,7 @@ type: metrics title: Logstash Node Stats dataset: logstash.node +release: experimental elasticsearch: index_template: mappings: @@ -8,8 +9,8 @@ elasticsearch: streams: - input: cel enabled: false - title: "Collect Node Stats" - description: "Collect Node Stats." + title: "Collect Node Metrics" + description: "Collect Metrics related to Nodes running Logstash" template_path: stream.yml.hbs vars: - name: period diff --git a/packages/logstash/data_stream/node_cel/sample_event.json b/packages/logstash/data_stream/node_cel/sample_event.json index eb3ed4c095b..dd5d35ff4ae 100644 --- a/packages/logstash/data_stream/node_cel/sample_event.json +++ b/packages/logstash/data_stream/node_cel/sample_event.json @@ -1,107 +1,19 @@ { - "@timestamp": "2023-10-02T20:29:51.743Z", - "agent": { - "name": "docker-fleet-agent", - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", - "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", - "type": "filebeat", - "version": "8.10.1" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node" - }, - "ecs": { - "version": "8.0.0" - }, - "elastic_agent": { - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", - "version": "8.10.1", - "snapshot": false - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-02T20:29:52Z", - "dataset": "logstash.node" - }, - "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - }, - "ip": [ - "192.168.86.57" - ], - "name": "docker-fleet-agent", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "mac": [ - "82-C1-07-06-9C-04" - ], - "architecture": "x86_64" - }, - "input": { - "type": "cel" - }, "logstash": { "node": { "stats": { "jvm": { "mem": { "heap_committed_in_bytes": 1073741824, - "heap_used_percent": 13, + "heap_used_percent": 14, "heap_max_in_bytes": 1073741822, - "non_heap_committed_in_bytes": 200540160, - "pools": { - "young": { - "used_in_bytes": 3145728, - "peak_used_in_bytes": 629145600, - "committed_in_bytes": 649068544, - "max_in_bytes": -1, - "peak_max_in_bytes": -1 - }, - "old": { - "used_in_bytes": 111104000, - "peak_used_in_bytes": 111104000, - "committed_in_bytes": 397410304, - "max_in_bytes": 1073741824, - "peak_max_in_bytes": 1073741824 - }, - "survivor": { - "used_in_bytes": 26945264, - "peak_used_in_bytes": 39405808, - "committed_in_bytes": 27262976, - "max_in_bytes": -1, - "peak_max_in_bytes": -1 - } - }, - "non_heap_used_in_bytes": 180942928, - "heap_used_in_bytes": 141194992 + "non_heap_committed_in_bytes": 161742848, + "heap_used_in_bytes": 151451136, + "non_heap_used_in_bytes": 152187584 }, "threads": { - "count": 46, + "count": 44, "peak_count": 50 - }, - "uptime_in_millis": 26669940, - "gc": { - "collectors": { - "young": { - "collection_count": 23, - "collection_time_in_millis": 341 - }, - "old": { - "collection_count": 0, - "collection_time_in_millis": 0 - } - } } }, "logstash": { @@ -115,61 +27,104 @@ "source" ], "http_address": "127.0.0.1:9600", + "host": "MacBook-Pro.local", "name": "rb", - "host": "robbaveys-MacBook-Pro.local", - "ephemeral_id": "43a7fd83-47ce-4646-bbb1-e93391041ada", - "version": "8.8.2", + "ephemeral_id": "41b5aa37-2e35-4054-bad6-56b517c22b00", "uuid": "b924ad43-2dbe-42b4-8931-75a52aa7fcbf", + "version": "8.8.2", "snapshot": false, "status": "green" }, "process": { - "open_file_descriptors": 104, "mem": { - "total_virtual_in_bytes": 40497684480 + "total_virtual_in_bytes": 40270282752 }, + "open_file_descriptors": 101, "max_file_descriptors": 10240, "cpu": { "load_average": { - "1m": 7.01220703125 + "1m": 4.91650390625 }, - "total_in_millis": 446183, + "total_in_millis": 61419, "percent": 0 }, - "peak_open_file_descriptors": 106 + "peak_open_file_descriptors": 105 }, "os": { - "dpu": { - "load_average": { - "1m": 7.01220703125 - }, - "percent": 0 - }, "cpu": { "load_average": { - "1m": 7.01220703125 + "1m": 4.91650390625 }, - "total_in_millis": 446183, + "total_in_millis": 61419, "percent": 0 }, "cgroup": {} }, - "queue": { - "events_count": 0 - }, "events": { "filtered": 0, "in": 0, "queue_push_duration_in_millis": 0, - "duration_in_millis": 1108, + "duration_in_millis": 61, "out": 0 }, + "queue": { + "events_count": 0 + }, "reloads": { "failures": 0, "successes": 0 } } } + }, + "input": { + "type": "cel" + }, + "agent": { + "name": "MacBook-Pro.local", + "id": "22c7007f-bdf3-4aaf-97bd-b01a5e9951e1", + "ephemeral_id": "966b3a32-dec5-41ab-94e4-aba8868c29b1", + "type": "filebeat", + "version": "8.10.1" + }, + "@timestamp": "2023-10-03T20:59:28.221Z", + "ecs": { + "version": "8.0.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.node" + }, + "host": { + "hostname": "macbook-pro.local", + "os": { + "build": "22F82", + "kernel": "22.5.0", + "name": "macOS", + "type": "macos", + "family": "darwin", + "version": "13.4.1", + "platform": "darwin" + }, + "ip": [ + "192.168.1.184" + ], + "name": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "82-C1-07-06-9C-00" + ], + "architecture": "x86_64" + }, + "elastic_agent": { + "id": "22c7007f-bdf3-4aaf-97bd-b01a5e9951e1", + "version": "8.10.1", + "snapshot": false + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-03T20:59:30Z", + "dataset": "logstash.node" } - } \ No newline at end of file diff --git a/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml index f0ab82be3a5..efeb3b64980 100644 --- a/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/pipeline/_dev/test/system/test-default-config.yml @@ -1,6 +1,6 @@ type: cel dataset: logstash.pipeline vars: - hosts: + url: - "http://{{Hostname}}:9600" data_stream: ~ diff --git a/packages/logstash/data_stream/pipeline/fields/fields.yml b/packages/logstash/data_stream/pipeline/fields/fields.yml index f28d795b82e..485455603ef 100644 --- a/packages/logstash/data_stream/pipeline/fields/fields.yml +++ b/packages/logstash/data_stream/pipeline/fields/fields.yml @@ -42,10 +42,10 @@ unit: ms type: long description: Time spent pushing events to the queue for this pipeline. - - name: reloads.success + - name: reloads.successes type: long description: Number of successful reloads for this pipeline - - name: reloads.failed + - name: reloads.failures type: long description: Number of failed reloads for this pipeline - name: flow @@ -56,10 +56,18 @@ type: scaled_float metric_type: gauge description: current value of the queue persisted growth events flow metric + - name: queue_persisted_growth_events.last_1_minute + type: scaled_float + metric_type: gauge + description: current value of the queue persisted growth events flow metric - name: queue_persisted_growth_bytes.current type: scaled_float metric_type: gauge description: current value of the queue persisted growth bytes flow metric + - name: queue_persisted_growth_bytes.last_1_minute + type: scaled_float + metric_type: gauge + description: current value of the queue persisted growth bytes flow metric - name: worker_concurrency.current type: scaled_float metric_type: gauge diff --git a/packages/logstash/data_stream/pipeline/fields/package-fields.yml b/packages/logstash/data_stream/pipeline/fields/package-fields.yml index 0e82e9b3a55..551dcda6414 100644 --- a/packages/logstash/data_stream/pipeline/fields/package-fields.yml +++ b/packages/logstash/data_stream/pipeline/fields/package-fields.yml @@ -1,3 +1,5 @@ +- name: input.type + type: keyword - name: cluster_uuid type: alias path: logstash.pipeline.elasticsearch.cluster.id diff --git a/packages/logstash/data_stream/pipeline/manifest.yml b/packages/logstash/data_stream/pipeline/manifest.yml index d13cc47d3b6..b10d2e7396b 100644 --- a/packages/logstash/data_stream/pipeline/manifest.yml +++ b/packages/logstash/data_stream/pipeline/manifest.yml @@ -1,5 +1,6 @@ type: metrics title: Logstash pipeline +release: experimental elasticsearch: index_template: mappings: @@ -7,8 +8,8 @@ elasticsearch: streams: - input: cel enabled: false - title: "Collect Pipeline Stats" - description: "Collect Pipeline Stats" + title: "Collect Pipeline Metrics" + description: "Collect Metrics related to Logstash Pipeline usage" template_path: stream.yml.hbs vars: - name: period diff --git a/packages/logstash/data_stream/pipeline/sample_event.json b/packages/logstash/data_stream/pipeline/sample_event.json index 1e6ebfd2723..af0fd83ab54 100644 --- a/packages/logstash/data_stream/pipeline/sample_event.json +++ b/packages/logstash/data_stream/pipeline/sample_event.json @@ -1,76 +1,21 @@ { - "@timestamp": "2023-10-02T20:29:51.743Z", - "agent": { - "name": "docker-fleet-agent", - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", - "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", - "type": "filebeat", - "version": "8.10.1" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.pipeline" - }, - "ecs": { - "version": "8.0.0" - }, - "elastic_agent": { - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", - "version": "8.10.1", - "snapshot": false - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-02T20:53:04Z", - "dataset": "logstash.pipeline" - }, - "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - }, - "ip": [ - "192.168.86.57" - ], - "name": "docker-fleet-agent", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "mac": [ - "82-C1-07-06-9C-04" - ], - "architecture": "x86_64" - }, - "input": { - "type": "cel" - }, "logstash": { "pipeline": { - "elasticsearch.cluster.id": [ - "t-wLhY6VT-SlT4zZ3tGLzg", - "9MOGoKiESvaklNVmxLo3iA" - ], + "elasticsearch.cluster.id": [], "total": { "queues": { "current_size": { "bytes": 1 }, "type": "persisted", - "events": 0, "max_size": { "bytes": 1073741824 - } + }, + "events": 0 }, "time": { "duration": { - "ms": 417 + "ms": 2 }, "queue_push_duration": { "ms": 0 @@ -78,32 +23,25 @@ }, "flow": { "worker_concurrency": { - "current": 0, - "last_1_minute": 0 + "current": 0.00009574 }, "queue_backpressure": { - "current": 0, - "last_1_minute": 0 + "current": 0 }, "input_throughput": { - "current": 0, - "last_1_minute": 0 + "current": 0 }, "queue_persisted_growth_events": { - "current": 0, - "last_1_minute": 0 + "current": 0 }, "queue_persisted_growth_bytes": { - "current": 0, - "last_1_minute": 0 + "current": 0 }, "filter_throughput": { - "current": 0, - "last_1_minute": 0 + "current": 0 }, "output_throughput": { - "current": 0, - "last_1_minute": 0 + "current": 0 } }, "events": { @@ -116,11 +54,61 @@ "successes": 0 } }, + "name": "source", "host": { "address": "127.0.0.1:9600", "name": "rb" - }, - "name": "sink" + } } + }, + "input": { + "type": "cel" + }, + "agent": { + "name": "MacBook-Pro.local", + "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", + "ephemeral_id": "1fea7ffc-92ee-4a81-9402-10ef28b471f8", + "type": "filebeat", + "version": "8.10.1" + }, + "@timestamp": "2023-10-03T20:37:08.677Z", + "ecs": { + "version": "8.0.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.pipeline" + }, + "host": { + "hostname": "macbook-pro.local", + "os": { + "build": "22F82", + "kernel": "22.5.0", + "name": "macOS", + "family": "darwin", + "type": "macos", + "version": "13.4.1", + "platform": "darwin" + }, + "ip": [ + "192.168.1.184" + ], + "name": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "82-C1-07-06-9C-00" + ], + "architecture": "x86_64" + }, + "elastic_agent": { + "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", + "version": "8.10.1", + "snapshot": false + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-03T20:37:09Z", + "dataset": "logstash.pipeline" } } \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml index d2e5279d470..efeb3b64980 100644 --- a/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml +++ b/packages/logstash/data_stream/plugins/_dev/test/system/test-default-config.yml @@ -1,6 +1,6 @@ -type: logstash/metrics +type: cel dataset: logstash.pipeline vars: - hosts: + url: - "http://{{Hostname}}:9600" data_stream: ~ diff --git a/packages/logstash/data_stream/plugins/fields/package-fields.yml b/packages/logstash/data_stream/plugins/fields/package-fields.yml index b0c006c06ec..9f878fbcda1 100644 --- a/packages/logstash/data_stream/plugins/fields/package-fields.yml +++ b/packages/logstash/data_stream/plugins/fields/package-fields.yml @@ -1,3 +1,5 @@ +- name: input.type + type: keyword - name: cluster_uuid type: alias path: logstash.pipeline.elasticsearch.cluster.id diff --git a/packages/logstash/data_stream/plugins/manifest.yml b/packages/logstash/data_stream/plugins/manifest.yml index 8961e2bb9a7..7de2a526bbb 100644 --- a/packages/logstash/data_stream/plugins/manifest.yml +++ b/packages/logstash/data_stream/plugins/manifest.yml @@ -1,5 +1,6 @@ type: metrics -title: Logstash pipeline +title: Logstash plugins +release: experimental elasticsearch: index_template: mappings: @@ -7,8 +8,8 @@ elasticsearch: streams: - input: cel enabled: false - title: "Collect Plugin Stats" - description: "Collect Plugin Stats" + title: "Collect Plugin Metrics" + description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate" template_path: stream.yml.hbs vars: - name: period diff --git a/packages/logstash/data_stream/plugins/sample_event.json b/packages/logstash/data_stream/plugins/sample_event.json index 9a9dbf11c3a..8846517955d 100644 --- a/packages/logstash/data_stream/plugins/sample_event.json +++ b/packages/logstash/data_stream/plugins/sample_event.json @@ -1,96 +1,86 @@ { - "@timestamp": "2023-10-02T20:29:51.743Z", + "logstash": { + "pipeline": { + "elasticsearch.cluster.id": [ + "_W1QSt-NRWGRG6nNFYxxZg", + "9MOGoKiESvaklNVmxLo3iA" + ], + "plugin": { + "input": { + "name": "pipeline", + "time": { + "queue_push_duration": { + "ms": 0 + } + }, + "id": "a30c4938ad8deb314752397cfe6f356b0441dfb67bf10f3ebaad455596b164e6", + "flow": { + "throughput": { + "current": 0 + } + }, + "events": { + "out": 0 + } + }, + "type": "input" + }, + "name": "sink", + "host": { + "address": "127.0.0.1:9600", + "name": "rb" + }, + "id": "81e8debdd29bc61f6416dcb7d305f8da144f2e15ae10a6d37d75d231100c8441" + } + }, + "input": { + "type": "cel" + }, "agent": { - "name": "docker-fleet-agent", - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", - "ephemeral_id": "0b015609-506e-4ff4-8ebc-0d8515ffc391", + "name": "MacBook-Pro.local", + "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", + "ephemeral_id": "1fea7ffc-92ee-4a81-9402-10ef28b471f8", "type": "filebeat", "version": "8.10.1" }, + "@timestamp": "2023-10-03T20:37:08.739Z", + "ecs": { + "version": "8.0.0" + }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "logstash.plugins" }, - "ecs": { - "version": "8.0.0" - }, "elastic_agent": { - "id": "0a7a4534-aaaf-41e2-9b5e-d7e7e69eceb0", + "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", "version": "8.10.1", "snapshot": false }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-02T20:50:52Z", - "dataset": "logstash.plugins" - }, "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", + "hostname": "macbook-pro.local", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "22F82", + "kernel": "22.5.0", + "name": "macOS", + "type": "macos", + "family": "darwin", + "version": "13.4.1", + "platform": "darwin" }, "ip": [ - "192.168.86.57" + "192.168.1.184" ], - "name": "docker-fleet-agent", + "name": "macbook-pro.local", "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", "mac": [ - "82-C1-07-06-9C-04" + "82-C1-07-06-9C-00" ], "architecture": "x86_64" }, - "input": { - "type": "cel" - }, - "logstash": { - "pipeline": { - "elasticsearch.cluster.id": [ - "9MOGoKiESvaklNVmxLo3iA" - ], - "plugin": { - "output": { - "elasticsearch.cluster.id": [ - "9MOGoKiESvaklNVmxLo3iA" - ], - "name": "elasticsearch", - "time": { - "duration": { - "ms": 90332 - } - }, - "id": "09a1c3091b4b30005c1d9f295fefab6072135a7064d8ccfb1171511e91c99c13", - "events": { - "in": 10252, - "out": 10252 - }, - "flow": { - "worker_millis_per_event": { - "current": 6.091, - "last_1_minute": 7.545 - }, - "worker_utilization": { - "current": 0.02805, - "last_1_minute": 0.01597 - } - } - }, - "type": "output" - }, - "name": "conversion_with_queues", - "host": { - "address": "127.0.0.1:9602", - "name": "rb2" - }, - "id": "75cb8b8394ad9d5eacdaa58768b827ba74103f41dcbf8d261e57cc2f9087261e" - } + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-03T20:37:09Z", + "dataset": "logstash.plugins" } } \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index c2f399acf3d..6e6ab181ade 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1156,7 +1156,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T14:11:03.240Z", + "created_at": "2023-10-03T21:22:03.327Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index 6c04491d3cc..2da42cfc2df 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -539,7 +539,7 @@ "query": "\"logstash.node.stats.jvm.mem.heap_used_in_bytes\": *" }, "isBucketed": false, - "label": "JVM Heap (MB)", + "label": "JVM Heap used", "operationType": "last_value", "params": { "format": { @@ -655,7 +655,8 @@ "yLeft": true, "yRight": true }, - "valueLabels": "hide" + "valueLabels": "hide", + "yTitle": "Heap Memory" } }, "title": "", @@ -1569,7 +1570,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T19:56:39.887Z", + "created_at": "2023-10-03T21:30:26.635Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", "managed": false, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index 2490aa9c8b3..6fced6e6c12 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -51,7 +51,7 @@ } }, "gridData": { - "h": 44, + "h": 54, "i": "231de1ff-a22e-495c-9c97-20b20592ac46", "w": 8, "x": 0, @@ -352,7 +352,6 @@ { "embeddableConfig": { "attributes": { - "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -368,26 +367,9 @@ "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ "467daee5-8948-4576-80fc-12bb76b01c0f", - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "8b34584c-a413-4629-898d-cda409db5fe0" ], "columns": { - "38a33c26-755e-4cf2-a5f2-739bf038af97": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.node.stats.jvm.threads.count\": *" - }, - "isBucketed": false, - "label": "Threads in use", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.jvm.threads.count" - }, "467daee5-8948-4576-80fc-12bb76b01c0f": { "dataType": "date", "isBucketed": true, @@ -401,21 +383,27 @@ "scale": "interval", "sourceField": "@timestamp" }, - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "8b34584c-a413-4629-898d-cda409db5fe0": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "\"logstash.node.stats.jvm.threads.peak_count\": *" + "query": "logstash.node.stats.os.cpu.percent: *" }, "isBucketed": false, - "label": "Peak threads in use", + "label": "OS CPU Utilization (%)", "operationType": "last_value", "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.jvm.threads.peak_count" + "sourceField": "logstash.node.stats.os.cpu.percent" } }, "incompleteColumns": {}, @@ -453,8 +441,7 @@ "layers": [ { "accessors": [ - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "8b34584c-a413-4629-898d-cda409db5fe0" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -463,20 +450,16 @@ "showGridlines": false, "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", "yConfig": [ - { - "color": "#3078c0", - "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" - }, { "color": "#ff0000", - "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" } ] } ], "legend": { "isVisible": true, - "position": "bottom" + "position": "right" }, "preferredSeriesType": "line", "tickLabelsVisibilitySettings": { @@ -496,19 +479,20 @@ }, "gridData": { "h": 10, - "i": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", + "i": "494716ff-128d-4a72-bf41-2f6f42dd5158", "w": 20, "x": 8, "y": 4 }, - "panelIndex": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", - "title": "Number of Java Threads", + "panelIndex": "494716ff-128d-4a72-bf41-2f6f42dd5158", + "title": "Process CPU Utilization (%)", "type": "lens", "version": "8.10.1" }, { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -524,7 +508,9 @@ "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ "467daee5-8948-4576-80fc-12bb76b01c0f", - "8b34584c-a413-4629-898d-cda409db5fe0" + "f85b46f2-de48-4729-90f4-8f254e25bede", + "a35f3d20-5df8-42b3-8dde-1e8bc9a8c6ba", + "a0dda6fc-ba9d-4edd-8058-ea5c3fac0eda" ], "columns": { "467daee5-8948-4576-80fc-12bb76b01c0f": { @@ -540,15 +526,31 @@ "scale": "interval", "sourceField": "@timestamp" }, - "8b34584c-a413-4629-898d-cda409db5fe0": { + "a0dda6fc-ba9d-4edd-8058-ea5c3fac0eda": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cpu.percent: *" + "query": "\"logstash.node.stats.os.cpu.load_average.15m\": *" }, "isBucketed": false, - "label": "OS CPU Utilization (%)", + "label": "Load Average (15m)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.os.cpu.load_average.15m" + }, + "a35f3d20-5df8-42b3-8dde-1e8bc9a8c6ba": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.os.cpu.load_average.5m\": *" + }, + "isBucketed": false, + "label": "Load Average (5m)", "operationType": "last_value", "params": { "format": { @@ -560,7 +562,29 @@ "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cpu.percent" + "sourceField": "logstash.node.stats.os.cpu.load_average.5m" + }, + "f85b46f2-de48-4729-90f4-8f254e25bede": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.process.cpu.load_average.1m\": *" + }, + "isBucketed": false, + "label": "Load Average (1m)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.cpu.load_average.1m" } }, "incompleteColumns": {}, @@ -598,7 +622,9 @@ "layers": [ { "accessors": [ - "8b34584c-a413-4629-898d-cda409db5fe0" + "f85b46f2-de48-4729-90f4-8f254e25bede", + "a35f3d20-5df8-42b3-8dde-1e8bc9a8c6ba", + "a0dda6fc-ba9d-4edd-8058-ea5c3fac0eda" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -606,12 +632,7 @@ "seriesType": "line", "showGridlines": false, "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", - "yConfig": [ - { - "color": "#ff0000", - "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" - } - ] + "yConfig": [] } ], "legend": { @@ -624,7 +645,8 @@ "yLeft": true, "yRight": true }, - "valueLabels": "hide" + "valueLabels": "hide", + "yTitle": "Load Average" } }, "title": "", @@ -636,13 +658,13 @@ }, "gridData": { "h": 10, - "i": "494716ff-128d-4a72-bf41-2f6f42dd5158", + "i": "a9970dca-d35d-4995-80aa-27d72da2cc33", "w": 20, "x": 28, "y": 4 }, - "panelIndex": "494716ff-128d-4a72-bf41-2f6f42dd5158", - "title": "Process CPU Utilization (%)", + "panelIndex": "a9970dca-d35d-4995-80aa-27d72da2cc33", + "title": "Load Average", "type": "lens", "version": "8.10.1" }, @@ -889,10 +911,11 @@ { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-336e8749-bfb1-4c0e-9f41-3892faebc198", + "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" } ], @@ -901,72 +924,57 @@ "datasourceStates": { "formBased": { "layers": { - "336e8749-bfb1-4c0e-9f41-3892faebc198": { + "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ - "95800073-8c87-4fee-8e70-b3ed0af3931b", - "51d651dd-246b-40fc-aca2-f9e3cb50e23f", - "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" + "467daee5-8948-4576-80fc-12bb76b01c0f", + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" ], "columns": { - "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b": { + "38a33c26-755e-4cf2-a5f2-739bf038af97": { + "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.pipeline.total.queues.events: *" + "query": "\"logstash.node.stats.jvm.threads.count\": *" }, "isBucketed": false, - "label": "Last value of logstash.pipeline.total.queues.events", + "label": "Threads in use", "operationType": "last_value", "params": { - "format": { - "id": "number", - "params": { - "decimals": 0, - "suffix": "" - } - }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.pipeline.total.queues.events" + "sourceField": "logstash.node.stats.jvm.threads.count" }, - "51d651dd-246b-40fc-aca2-f9e3cb50e23f": { + "467daee5-8948-4576-80fc-12bb76b01c0f": { "dataType": "date", "isBucketed": true, "label": "@timestamp", "operationType": "date_histogram", "params": { - "dropPartials": true, + "dropPartials": false, "includeEmptyRows": true, "interval": "auto" }, "scale": "interval", "sourceField": "@timestamp" }, - "95800073-8c87-4fee-8e70-b3ed0af3931b": { - "dataType": "string", - "isBucketed": true, - "label": "Top 10 values of logstash.pipeline.name", - "operationType": "terms", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.jvm.threads.peak_count\": *" + }, + "isBucketed": false, + "label": "Peak threads in use", + "operationType": "last_value", "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 10 + "sortField": "@timestamp" }, - "scale": "ordinal", - "sourceField": "logstash.pipeline.name" + "scale": "ratio", + "sourceField": "logstash.node.stats.jvm.threads.peak_count" } }, "incompleteColumns": {}, @@ -1004,25 +1012,28 @@ "layers": [ { "accessors": [ - "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" ], - "layerId": "336e8749-bfb1-4c0e-9f41-3892faebc198", + "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", "position": "top", "seriesType": "line", "showGridlines": false, - "splitAccessor": "95800073-8c87-4fee-8e70-b3ed0af3931b", - "xAccessor": "51d651dd-246b-40fc-aca2-f9e3cb50e23f", + "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", "yConfig": [ + { + "color": "#3078c0", + "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" + }, { "color": "#ff0000", - "forAccessor": "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" + "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" } ] } ], "legend": { - "isInside": false, "isVisible": true, "position": "bottom" }, @@ -1032,10 +1043,7 @@ "yLeft": true, "yRight": true }, - "valueLabels": "hide", - "yLeftExtent": { - "mode": "dataBounds" - } + "valueLabels": "hide" } }, "title": "", @@ -1047,13 +1055,13 @@ }, "gridData": { "h": 10, - "i": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", + "i": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", "w": 20, "x": 28, - "y": 14 + "y": 24 }, - "panelIndex": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", - "title": "Persistent Queue Events", + "panelIndex": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360", + "title": "Number of Java Threads", "type": "lens", "version": "8.10.1" }, @@ -1203,7 +1211,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "indexpattern-datasource-layer-336e8749-bfb1-4c0e-9f41-3892faebc198", "type": "index-pattern" } ], @@ -1212,35 +1220,21 @@ "datasourceStates": { "formBased": { "layers": { - "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { + "336e8749-bfb1-4c0e-9f41-3892faebc198": { "columnOrder": [ - "467daee5-8948-4576-80fc-12bb76b01c0f", - "8b34584c-a413-4629-898d-cda409db5fe0", - "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + "95800073-8c87-4fee-8e70-b3ed0af3931b", + "51d651dd-246b-40fc-aca2-f9e3cb50e23f", + "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" ], "columns": { - "467daee5-8948-4576-80fc-12bb76b01c0f": { - "dataType": "date", - "isBucketed": true, - "label": "@timestamp", - "operationType": "date_histogram", - "params": { - "dropPartials": false, - "includeEmptyRows": true, - "interval": "auto" - }, - "scale": "interval", - "sourceField": "@timestamp" - }, - "4c49f25e-d518-4d91-bcdc-69ea91729ca3": { - "customLabel": true, + "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b": { "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled: *" + "query": "logstash.pipeline.total.queues.events: *" }, "isBucketed": false, - "label": "CGroup Throttled Count", + "label": "Last value of logstash.pipeline.total.queues.events", "operationType": "last_value", "params": { "format": { @@ -1253,33 +1247,48 @@ "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled" + "sourceField": "logstash.pipeline.total.queues.events" }, - "8b34584c-a413-4629-898d-cda409db5fe0": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods: *" - }, - "isBucketed": false, - "label": "CGroup Elapsed Periods", - "operationType": "last_value", + "51d651dd-246b-40fc-aca2-f9e3cb50e23f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", "params": { - "format": { - "id": "number", - "params": { - "decimals": 0, - "suffix": "" - } - }, - "sortField": "@timestamp" + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" }, - "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods" - } - }, - "incompleteColumns": {}, + "scale": "interval", + "sourceField": "@timestamp" + }, + "95800073-8c87-4fee-8e70-b3ed0af3931b": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of logstash.pipeline.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + } + }, + "incompleteColumns": {}, "sampling": 1 } } @@ -1314,8 +1323,183 @@ "layers": [ { "accessors": [ - "8b34584c-a413-4629-898d-cda409db5fe0", - "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" + ], + "layerId": "336e8749-bfb1-4c0e-9f41-3892faebc198", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "95800073-8c87-4fee-8e70-b3ed0af3931b", + "xAccessor": "51d651dd-246b-40fc-aca2-f9e3cb50e23f", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b" + } + ] + } + ], + "legend": { + "isInside": false, + "isVisible": true, + "position": "bottom" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 10, + "i": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", + "w": 20, + "x": 28, + "y": 14 + }, + "panelIndex": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d", + "title": "Persistent Queue Events", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { + "columnOrder": [ + "467daee5-8948-4576-80fc-12bb76b01c0f", + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729", + "2ed07426-3ab6-4f45-aaf7-15934ba2d7ec" + ], + "columns": { + "2ed07426-3ab6-4f45-aaf7-15934ba2d7ec": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.process.peak_open_file_descriptors\": *" + }, + "isBucketed": false, + "label": "Peak open file descriptors", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.peak_open_file_descriptors" + }, + "38a33c26-755e-4cf2-a5f2-739bf038af97": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.process.open_file_descriptors: *" + }, + "isBucketed": false, + "label": "File Descriptors in use", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.open_file_descriptors" + }, + "467daee5-8948-4576-80fc-12bb76b01c0f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.process.max_file_descriptors: *" + }, + "isBucketed": false, + "label": "Max File Descriptors", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.process.max_file_descriptors" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "38a33c26-755e-4cf2-a5f2-739bf038af97", + "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729", + "2ed07426-3ab6-4f45-aaf7-15934ba2d7ec" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -1325,12 +1509,12 @@ "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", "yConfig": [ { - "color": "#ff0000", - "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" + "color": "#3078c0", + "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" }, { - "color": "#6092c0", - "forAccessor": "4c49f25e-d518-4d91-bcdc-69ea91729ca3" + "color": "#ff0000", + "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" } ] } @@ -1357,13 +1541,13 @@ }, "gridData": { "h": 10, - "i": "71538b79-3608-4191-b3eb-0f898b7d7a3b", + "i": "92979e91-3f35-4715-b97a-f088bb2eae51", "w": 20, - "x": 28, - "y": 24 + "x": 8, + "y": 34 }, - "panelIndex": "71538b79-3608-4191-b3eb-0f898b7d7a3b", - "title": "CGroup CFS Stats", + "panelIndex": "92979e91-3f35-4715-b97a-f088bb2eae51", + "title": "Number of File Descriptors", "type": "lens", "version": "8.10.1" }, @@ -1407,46 +1591,46 @@ "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos: *" + "query": "logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled: *" }, "isBucketed": false, - "label": "CGroup Throttling (ns)", + "label": "CGroup Throttled Count", "operationType": "last_value", "params": { "format": { "id": "number", "params": { - "decimals": 1, - "suffix": "ns" + "decimals": 0, + "suffix": "" } }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos" + "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled" }, "8b34584c-a413-4629-898d-cda409db5fe0": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos: *" + "query": "logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods: *" }, "isBucketed": false, - "label": "CGroup CPU Performance (ns)", + "label": "CGroup Elapsed Periods", "operationType": "last_value", "params": { "format": { "id": "number", "params": { - "decimals": 1, - "suffix": "ns" + "decimals": 0, + "suffix": "" } }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos" + "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods" } }, "incompleteColumns": {}, @@ -1527,13 +1711,13 @@ }, "gridData": { "h": 10, - "i": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", + "i": "71538b79-3608-4191-b3eb-0f898b7d7a3b", "w": 20, "x": 28, "y": 34 }, - "panelIndex": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", - "title": "CGroup Cpu Performance (ns)", + "panelIndex": "71538b79-3608-4191-b3eb-0f898b7d7a3b", + "title": "CGroup CFS Stats", "type": "lens", "version": "8.10.1" }, @@ -1555,26 +1739,10 @@ "c03b1610-b629-4e6c-ae2a-0e460cd446d2": { "columnOrder": [ "467daee5-8948-4576-80fc-12bb76b01c0f", - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "8b34584c-a413-4629-898d-cda409db5fe0", + "4c49f25e-d518-4d91-bcdc-69ea91729ca3" ], "columns": { - "38a33c26-755e-4cf2-a5f2-739bf038af97": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.node.stats.process.open_file_descriptors: *" - }, - "isBucketed": false, - "label": "File Descriptors in use", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.node.stats.process.open_file_descriptors" - }, "467daee5-8948-4576-80fc-12bb76b01c0f": { "dataType": "date", "isBucketed": true, @@ -1588,21 +1756,51 @@ "scale": "interval", "sourceField": "@timestamp" }, - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729": { + "4c49f25e-d518-4d91-bcdc-69ea91729ca3": { "customLabel": true, "dataType": "number", "filter": { "language": "kuery", - "query": "logstash.node.stats.process.max_file_descriptors: *" + "query": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos: *" }, "isBucketed": false, - "label": "Max File Descriptors", + "label": "CGroup Throttling (ns)", "operationType": "last_value", "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": "ns" + } + }, "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.node.stats.process.max_file_descriptors" + "sourceField": "logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos" + }, + "8b34584c-a413-4629-898d-cda409db5fe0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos: *" + }, + "isBucketed": false, + "label": "CGroup CPU Performance (ns)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": "ns" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.os.cgroup.cpuacct.usage_nanos" } }, "incompleteColumns": {}, @@ -1640,8 +1838,8 @@ "layers": [ { "accessors": [ - "38a33c26-755e-4cf2-a5f2-739bf038af97", - "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "8b34584c-a413-4629-898d-cda409db5fe0", + "4c49f25e-d518-4d91-bcdc-69ea91729ca3" ], "layerId": "c03b1610-b629-4e6c-ae2a-0e460cd446d2", "layerType": "data", @@ -1651,12 +1849,12 @@ "xAccessor": "467daee5-8948-4576-80fc-12bb76b01c0f", "yConfig": [ { - "color": "#3078c0", - "forAccessor": "38a33c26-755e-4cf2-a5f2-739bf038af97" + "color": "#ff0000", + "forAccessor": "8b34584c-a413-4629-898d-cda409db5fe0" }, { - "color": "#ff0000", - "forAccessor": "d2fb1c2f-f8d2-4b82-8d99-4aabfc642729" + "color": "#6092c0", + "forAccessor": "4c49f25e-d518-4d91-bcdc-69ea91729ca3" } ] } @@ -1683,13 +1881,13 @@ }, "gridData": { "h": 10, - "i": "92979e91-3f35-4715-b97a-f088bb2eae51", + "i": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", "w": 20, - "x": 8, - "y": 34 + "x": 28, + "y": 44 }, - "panelIndex": "92979e91-3f35-4715-b97a-f088bb2eae51", - "title": "Number of File Descriptors", + "panelIndex": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1", + "title": "CGroup Cpu Performance (ns)", "type": "lens", "version": "8.10.1" } @@ -1699,7 +1897,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T21:52:39.962Z", + "created_at": "2023-10-03T21:31:06.493Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", "managed": false, "references": [ @@ -1710,12 +1908,12 @@ }, { "id": "logstash-sm-metrics", - "name": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "494716ff-128d-4a72-bf41-2f6f42dd5158:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "494716ff-128d-4a72-bf41-2f6f42dd5158:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "a9970dca-d35d-4995-80aa-27d72da2cc33:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { @@ -1725,7 +1923,7 @@ }, { "id": "logstash-sm-metrics", - "name": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d:indexpattern-datasource-layer-336e8749-bfb1-4c0e-9f41-3892faebc198", + "name": "f2fcf8cd-8ef3-42b8-b90b-d113c5e92360:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { @@ -1735,17 +1933,22 @@ }, { "id": "logstash-sm-metrics", - "name": "71538b79-3608-4191-b3eb-0f898b7d7a3b:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "5af28ec8-a9f0-49cb-9627-e13c0ac5ca1d:indexpattern-datasource-layer-336e8749-bfb1-4c0e-9f41-3892faebc198", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "92979e91-3f35-4715-b97a-f088bb2eae51:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "92979e91-3f35-4715-b97a-f088bb2eae51:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "name": "71538b79-3608-4191-b3eb-0f898b7d7a3b:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "c6fb1dc0-c51d-4c00-903c-d90ad3b77ce1:indexpattern-datasource-layer-c03b1610-b629-4e6c-ae2a-0e460cd446d2", "type": "index-pattern" }, { diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index 3794733ff40..b3de737478e 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -41,7 +41,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipeline Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n**[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)** \n\n\nOverview\n\nThis Dashboard gives a detailed view of a Logstash Pipeline. Should be used in conjunction with pipeline and node controls", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipeline Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n**[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)** \n\n\nOverview\n\nThis Dashboard gives a detailed view of a Logstash pipeline. \nUse controls to drill down on pipeline performance on each pipeline, and on individual nodes running those pipelines.", "openLinksInNewTab": false }, "title": "", @@ -179,7 +179,7 @@ "gridData": { "h": 5, "i": "9a756479-5952-441e-beff-e0896500ca39", - "w": 14, + "w": 10, "x": 8, "y": 0 }, @@ -187,6 +187,116 @@ "type": "lens", "version": "8.10.1" }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "d6fe4fea-e92e-4042-9c92-0663b598a523": { + "columnOrder": [ + "27acc57b-d205-4010-a882-9ea93f8fc3ab", + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52" + ], + "columns": { + "27acc57b-d205-4010-a882-9ea93f8fc3ab": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of logstash.host.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "609dea7b-c32b-49bc-b5a7-af8fe67d2d52": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.total.events.in\": *" + }, + "isBucketed": false, + "label": "Events received by pipeline", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.in" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "27acc57b-d205-4010-a882-9ea93f8fc3ab", + "collapseFn": "sum", + "layerId": "d6fe4fea-e92e-4042-9c92-0663b598a523", + "layerType": "data", + "metricAccessor": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", + "showBar": false + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 5, + "i": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", + "w": 10, + "x": 18, + "y": 0 + }, + "panelIndex": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", + "type": "lens", + "version": "8.10.1" + }, { "embeddableConfig": { "attributes": { @@ -289,8 +399,8 @@ "gridData": { "h": 5, "i": "6da9466a-ede0-44b6-a2e7-d704065bb90b", - "w": 13, - "x": 35, + "w": 10, + "x": 28, "y": 0 }, "panelIndex": "6da9466a-ede0-44b6-a2e7-d704065bb90b", @@ -460,116 +570,6 @@ "type": "lens", "version": "8.10.1" }, - { - "embeddableConfig": { - "attributes": { - "description": "", - "references": [ - { - "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "d6fe4fea-e92e-4042-9c92-0663b598a523": { - "columnOrder": [ - "27acc57b-d205-4010-a882-9ea93f8fc3ab", - "609dea7b-c32b-49bc-b5a7-af8fe67d2d52" - ], - "columns": { - "27acc57b-d205-4010-a882-9ea93f8fc3ab": { - "dataType": "string", - "isBucketed": true, - "label": "Top 5 values of logstash.host.name", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 5 - }, - "scale": "ordinal", - "sourceField": "logstash.host.name" - }, - "609dea7b-c32b-49bc-b5a7-af8fe67d2d52": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.total.events.in\": *" - }, - "isBucketed": false, - "label": "Events received by pipeline", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.total.events.in" - } - }, - "ignoreGlobalFilters": false, - "incompleteColumns": {}, - "sampling": 1 - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "breakdownByAccessor": "27acc57b-d205-4010-a882-9ea93f8fc3ab", - "collapseFn": "sum", - "layerId": "d6fe4fea-e92e-4042-9c92-0663b598a523", - "layerType": "data", - "metricAccessor": "609dea7b-c32b-49bc-b5a7-af8fe67d2d52", - "showBar": false - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsMetric" - }, - "enhancements": {}, - "hidePanelTitles": true - }, - "gridData": { - "h": 5, - "i": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", - "w": 13, - "x": 22, - "y": 0 - }, - "panelIndex": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e", - "type": "lens", - "version": "8.10.1" - }, { "embeddableConfig": { "attributes": { @@ -6617,9 +6617,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T19:44:30.925Z", + "created_at": "2023-10-03T21:22:03.327Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", @@ -6633,17 +6633,17 @@ }, { "id": "logstash-sm-metrics", - "name": "6da9466a-ede0-44b6-a2e7-d704065bb90b:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "name": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "f03395de-a163-4204-ac82-83c0ea542eeb:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", + "name": "6da9466a-ede0-44b6-a2e7-d704065bb90b:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "b48c9b95-bcc4-4bde-b41d-630a4f31b26e:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", + "name": "f03395de-a163-4204-ac82-83c0ea542eeb:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", "type": "index-pattern" }, { diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index b682b5d8493..42d0ccbd1f1 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -1511,7 +1511,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n**[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548)** \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\n\nOverview\n\nThis Dashboard gives an overall view of Pipelines Running on Logstash", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n**[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548)** \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\n\nOverview\n\nThis Dashboard gives an overall view of pipelines running on dashboards. \nClick on a pipeline in the list view to get more details on the running pipeline, and the plugins in that pipeline.", "openLinksInNewTab": false }, "title": "", @@ -1537,9 +1537,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T19:05:10.115Z", + "created_at": "2023-10-03T21:22:03.327Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 32f568213ab..e14b046db0f 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -909,9 +909,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-02T19:47:14.298Z", + "created_at": "2023-10-03T21:22:03.327Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", From e5d96f09caa2ea564eb3a94635162a55db4f3a72 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 4 Oct 2023 12:20:59 -0400 Subject: [PATCH 07/25] Tidy and use time series indices --- .../data_stream/node_cel/fields/fields.yml | 90 ++++++++++--------- .../data_stream/node_cel/manifest.yml | 2 +- .../data_stream/pipeline/fields/fields.yml | 13 ++- .../data_stream/pipeline/manifest.yml | 1 + .../data_stream/plugins/fields/fields.yml | 28 ++++-- .../logstash/data_stream/plugins/manifest.yml | 1 + packages/logstash/validation.yml | 3 + 7 files changed, 88 insertions(+), 50 deletions(-) create mode 100644 packages/logstash/validation.yml diff --git a/packages/logstash/data_stream/node_cel/fields/fields.yml b/packages/logstash/data_stream/node_cel/fields/fields.yml index dc9f266a74e..408931027f9 100644 --- a/packages/logstash/data_stream/node_cel/fields/fields.yml +++ b/packages/logstash/data_stream/node_cel/fields/fields.yml @@ -6,13 +6,6 @@ - name: node type: group fields: - - name: state.pipeline - type: group - fields: - - name: id - type: keyword - - name: hash - type: keyword - name: stats type: group fields: @@ -21,8 +14,10 @@ fields: - name: failures type: long + metric_type: counter - name: successes type: long + metric_type: counter - name: timestamp type: date - name: jvm @@ -33,27 +28,36 @@ fields: - name: count type: long + metric_type: counter description: current number of threads - name: peak_count type: long + metric_type: counter description: peak number of threads - name: uptime_in_millis type: long + metric_type: counter - name: mem type: group fields: - name: heap_used_in_bytes type: long + metric_type: gauge - name: heap_max_in_bytes type: long + metric_type: counter - name: heap_used_percent type: long + metric_type: gauge - name: heap_committed_in_bytes type: long + metric_type: gauge - name: non_heap_committed_in_bytes type: long + metric_type: gauge - name: non_heap_used_in_bytes type: long + metric_type: gauge - name: gc type: group fields: @@ -65,34 +69,40 @@ fields: - name: collection_count type: long + metric_type: counter - name: collection_time_in_millis type: long + metric_type: counter - name: young type: group fields: - name: collection_count type: long + metric_type: counter - name: collection_time_in_millis type: long + metric_type: counter - name: events type: group fields: - name: in type: long - description: | - Incoming events counter. + metric_type: counter + description: Incoming events counter - name: out type: long - description: | - Outgoing events counter. + metric_type: counter + description: Outgoing events counter - name: filtered type: long - description: | - Filtered events counter. + metric_type: counter + description: Filtered events counter - name: duration_in_millis type: long + metric_type: counter - name: queue_push_duration_in_millis type: long + metric_type: counter - name: logstash type: group fields: @@ -102,6 +112,7 @@ type: keyword - name: ephemeral_id type: keyword + dimension: true - name: host type: keyword - name: http_address @@ -119,10 +130,13 @@ fields: - name: batch_size type: long + metric_type: gauge - name: batch_delay type: long + metric_type: gauge - name: workers type: long + metric_type: gauge - name: os type: group fields: @@ -131,17 +145,22 @@ fields: - name: percent type: double + metric_type: gauge - name: total_in_millis type: long + metric_type: counter - name: load_average type: group fields: - name: 15m type: half_float + metric_type: gauge - name: 1m type: half_float + metric_type: gauge - name: 5m type: half_float + metric_type: gauge - name: cgroup type: group fields: @@ -149,25 +168,30 @@ type: group fields: - name: control_group - type: text + type: keyword - name: usage_nanos type: long + metric_type: counter - name: cpu type: group fields: - name: control_group - type: text + type: keyword - name: cfs_quota_micros type: long + metric_type: gauge - name: stat type: group fields: - name: number_of_elapsed_periods type: long + metric_type: gauge - name: time_throttled_nanos type: long + metric_type: counter - name: number_of_times_throttled type: long + metric_type: counter - name: process type: group fields: @@ -176,29 +200,37 @@ fields: - name: percent type: double + metric_type: gauge - name: total_in_millis type: long + metric_type: counter - name: load_average type: group fields: - name: 15m type: half_float + metric_type: gauge - name: 1m type: half_float + metric_type: gauge - name: 5m type: half_float + metric_type: gauge - name: mem.total_virtual_in_bytes type: long + metric_type: gauge - name: max_file_descriptors type: long + metric_type: gauge - name: open_file_descriptors type: long + metric_type: gauge - name: peak_open_file_descriptors type: long + metric_type: gauge - name: queue.events_count type: long - - name: pipelines - type: nested + metric_type: counter - name: pipelines.id type: keyword - name: pipelines.hash @@ -236,27 +268,3 @@ type: long - name: queue_push_duration_in_millis type: long - - name: pipelines.vertices - type: nested - - name: pipelines.vertices.id - type: keyword - description: id - - name: pipelines.vertices.long_counters - type: nested - - name: pipelines.vertices.long_counters.name - type: keyword - - name: pipelines.vertices.long_counters.value - type: long - - name: pipelines.vertices.duration_in_millis - type: long - - name: pipelines.vertices.events_in - type: long - - name: pipelines.vertices.pipeline_ephemeral_id - type: keyword - description: pipeline_ephemeral_id - - name: pipelines.vertices.events_out - type: long - description: events_out - - name: pipelines.vertices.queue_push_duration_in_millis - type: long - description: queue_push_duration_in_millis diff --git a/packages/logstash/data_stream/node_cel/manifest.yml b/packages/logstash/data_stream/node_cel/manifest.yml index e2b87d773bc..e868d1097bf 100644 --- a/packages/logstash/data_stream/node_cel/manifest.yml +++ b/packages/logstash/data_stream/node_cel/manifest.yml @@ -5,7 +5,7 @@ release: experimental elasticsearch: index_template: mappings: - dynamic: true + dynamic: false streams: - input: cel enabled: false diff --git a/packages/logstash/data_stream/pipeline/fields/fields.yml b/packages/logstash/data_stream/pipeline/fields/fields.yml index 485455603ef..6ce6b92db9e 100644 --- a/packages/logstash/data_stream/pipeline/fields/fields.yml +++ b/packages/logstash/data_stream/pipeline/fields/fields.yml @@ -4,10 +4,8 @@ fields: - name: name type: keyword + dimension: true description: Logstash Pipeline id/name - - name: id - type: wildcard - description: Logstash Pipeline hash - name: elasticsearch.cluster.id type: keyword description: Elasticsearch clusters this Logstash pipeline is attached to @@ -21,6 +19,7 @@ - name: address type: keyword description: address hosting this instance of logstash + dimension: true - name: total type: group description: Total metrics on Logstash Pipeline @@ -28,25 +27,32 @@ - name: events.filtered type: long description: Number of events filtered by the pipeline + metric_type: counter - name: events.in type: long + metric_type: counter description: Number of events received by the pipeline - name: events.out type: long + metric_type: counter description: Number of events emitted by the pipeline - name: time.duration.ms unit: ms type: long + metric_type: counter description: Time spent processing events through the pipeline. - name: time.queue_push_duration.ms + metric_type: counter unit: ms type: long description: Time spent pushing events to the queue for this pipeline. - name: reloads.successes type: long + metric_type: counter description: Number of successful reloads for this pipeline - name: reloads.failures type: long + metric_type: counter description: Number of failed reloads for this pipeline - name: flow type: group @@ -117,6 +123,7 @@ - name: events type: long description: Number of events in the PQ for this pipeline + metric_type: counter - name: current_size.bytes type: long metric_type: gauge diff --git a/packages/logstash/data_stream/pipeline/manifest.yml b/packages/logstash/data_stream/pipeline/manifest.yml index b10d2e7396b..2aef1281593 100644 --- a/packages/logstash/data_stream/pipeline/manifest.yml +++ b/packages/logstash/data_stream/pipeline/manifest.yml @@ -2,6 +2,7 @@ type: metrics title: Logstash pipeline release: experimental elasticsearch: + index_mode: "time_series" index_template: mappings: dynamic: false diff --git a/packages/logstash/data_stream/plugins/fields/fields.yml b/packages/logstash/data_stream/plugins/fields/fields.yml index 2940dc07fc7..e898500dd4f 100644 --- a/packages/logstash/data_stream/plugins/fields/fields.yml +++ b/packages/logstash/data_stream/plugins/fields/fields.yml @@ -6,7 +6,8 @@ type: keyword description: Logstash Pipeline id/name - name: id - type: wildcard + type: keyword + dimension: true description: Logstash Pipeline hash - name: elasticsearch.cluster.id type: keyword @@ -36,24 +37,30 @@ type: keyword description: Name of codec plugin - name: id - type: wildcard + type: keyword + dimension: true description: Id of codec plugin - name: encode.duration.ms type: long description: amount of time spend encoding events unit: ms + metric_type: counter - name: encode.in type: long description: number of events encoded + metric_type: counter - name: decode.duration.ms type: long description: amount of time spend decoding events unit: ms + metric_type: counter - name: decode.in type: long description: number of events entering the decoder + metric_type: counter - name: decode.out type: long + metric_type: counter description: number of events exiting the decoder - name: input type: group @@ -63,17 +70,20 @@ type: keyword description: Name of input plugin - name: id - type: wildcard + type: keyword + dimension: true description: Id of input plugin - name: elasticsearch.cluster.id type: keyword description: Elasticsearch clusters this Logstash plugin is attached to - name: time.queue_push_duration.ms type: long + metric_type: counter description: amount of time spend pushing events to the queue unit: ms - name: events.out type: long + metric_type: counter description: number of events emitted by the input - name: flow type: group @@ -95,21 +105,25 @@ type: keyword description: Name of filter plugin - name: id - type: wildcard + type: keyword + dimension: true description: Id of filter plugin - name: elasticsearch.cluster.id type: keyword description: Elasticsearch clusters this Logstash plugin is attached to - name: time.duration.ms type: long + metric_type: counter description: amount of time working on events in this plugin unit: ms - name: events.in type: long description: number of events received by the filter + metric_type: counter - name: events.out type: long description: number of events emitted by the filter + metric_type: counter - name: flow type: group description: flow metrics @@ -138,20 +152,24 @@ type: keyword description: Name of output plugin - name: id - type: wildcard + type: keyword + dimension: true description: Id of output plugin - name: elasticsearch.cluster.id type: keyword description: Elasticsearch clusters this Logstash plugin is attached to - name: time.duration.ms type: long + metric_type: counter description: amount of time working on events in this plugin unit: ms - name: events.in type: long + metric_type: counter description: number of events received by the output - name: events.out type: long + metric_type: counter description: number of events emitted by the output - name: flow type: group diff --git a/packages/logstash/data_stream/plugins/manifest.yml b/packages/logstash/data_stream/plugins/manifest.yml index 7de2a526bbb..dc96ee85294 100644 --- a/packages/logstash/data_stream/plugins/manifest.yml +++ b/packages/logstash/data_stream/plugins/manifest.yml @@ -2,6 +2,7 @@ type: metrics title: Logstash plugins release: experimental elasticsearch: + index_mode: "time_series" index_template: mappings: dynamic: false diff --git a/packages/logstash/validation.yml b/packages/logstash/validation.yml new file mode 100644 index 00000000000..2527f20c354 --- /dev/null +++ b/packages/logstash/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - SVR00002 # Mandatory filters in dashboards. From f801aa70fd36e763109b3b69f38ac5e8af55c037 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 4 Oct 2023 15:40:19 -0400 Subject: [PATCH 08/25] Tidying, adding readme text --- packages/logstash/_dev/build/docs/README.md | 51 + .../node_cel/agent/stream/stream.yml.hbs | 21 +- .../data_stream/node_cel/fields/agent.yml | 141 ++ .../data_stream/node_cel/sample_event.json | 94 +- .../data_stream/pipeline/fields/agent.yml | 141 ++ .../data_stream/pipeline/sample_event.json | 172 +-- .../data_stream/plugins/fields/agent.yml | 141 ++ .../data_stream/plugins/sample_event.json | 128 +- packages/logstash/docs/README.md | 671 ++++++++ ...-79270240-48ee-11ee-8cb5-99927777c522.json | 2 +- ...-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 4 +- ...-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 4 +- ...-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 1358 +++++++++++------ ...-c0594170-526a-11ee-9ecc-31444cb79548.json | 641 ++++++-- ...-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 2 +- 15 files changed, 2771 insertions(+), 800 deletions(-) create mode 100644 packages/logstash/data_stream/node_cel/fields/agent.yml create mode 100644 packages/logstash/data_stream/pipeline/fields/agent.yml create mode 100644 packages/logstash/data_stream/plugins/fields/agent.yml diff --git a/packages/logstash/_dev/build/docs/README.md b/packages/logstash/_dev/build/docs/README.md index 6e428d5b495..7b28ed0020c 100644 --- a/packages/logstash/_dev/build/docs/README.md +++ b/packages/logstash/_dev/build/docs/README.md @@ -6,6 +6,15 @@ The `logstash` package collects metrics and logs of Logstash. The `logstash` package works with Logstash 8.5.0 and later +## Technical Preview note + +This Logstash package also includes a technical preview of Logstash data collection and dashboards +native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. + +Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, +and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` + + ## Logs Logstash package supports the plain text format and the JSON format. Also, two types of @@ -79,3 +88,45 @@ Logstash metric related data streams works with Logstash 7.3.0 and later. ### Node {{event "node"}} + + +## Metrics (Technical Preview) + +This Logstash package also includes a technical preview of Logstash data collection and dashboards +native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. + +Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, +and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` + +### Fields and Sample Event + +#### Node + +This is the `node` dataset, which drives the Node dashboard pages. + +#### Example + +{{fields "node_cel"}} + +{{event "node_cel"}} + +#### Pipeline + +This is the `pipeline` dataset, which drives the Pipeline dashboard pages. + +#### Example + +{{fields "pipeline"}} + +{{event "pipeline"}} + +#### Plugin + +This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the +pipeline collection period, and setting it to an appropriate value. + +#### Example + +{{fields "plugins"}} + +{{event "plugins"}} diff --git a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs index 11d5ffab915..dfe447a8749 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs @@ -33,8 +33,8 @@ program: | ) .do_request().as(resp, bytes(resp.Body) .decode_json() - .as(body, {"b":body} - .with({"logstash":{"node":{"stats":{ + .as(body, + {"logstash":{"node":{"stats":{ "events":body['events'], "jvm":{ "uptime_in_millis":body['jvm']['uptime_in_millis'], @@ -61,12 +61,11 @@ program: | "version":body['version'], } }} - }}) - ) - ).drop("b") - .as(eve, { - "events":eve.map(each, eve), - "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, - "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null - }, - }) \ No newline at end of file + }}) + ) + .as(eve, { + "events":eve.map(each, eve), + "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, + "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null + }, + }) \ No newline at end of file diff --git a/packages/logstash/data_stream/node_cel/fields/agent.yml b/packages/logstash/data_stream/node_cel/fields/agent.yml new file mode 100644 index 00000000000..d0032fb9fff --- /dev/null +++ b/packages/logstash/data_stream/node_cel/fields/agent.yml @@ -0,0 +1,141 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + - name: availability_zone + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + - name: instance.id + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + - name: instance.name + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + - name: provider + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + - name: region + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + type: object + object_type: keyword + description: Image labels. + - name: name + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + type: keyword + ignore_above: 1024 + description: Operating system architecture. + - name: domain + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + default_field: false + - name: hostname + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + type: ip + description: Host ip addresses. + - name: mac + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + - name: os.kernel + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + - name: os.name + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + - name: os.platform + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + - name: os.version + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + - name: type + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + description: > + OS build information. + + - name: os.codename + type: keyword + description: > + OS codename, if any. + diff --git a/packages/logstash/data_stream/node_cel/sample_event.json b/packages/logstash/data_stream/node_cel/sample_event.json index dd5d35ff4ae..370fc809ae5 100644 --- a/packages/logstash/data_stream/node_cel/sample_event.json +++ b/packages/logstash/data_stream/node_cel/sample_event.json @@ -4,68 +4,74 @@ "stats": { "jvm": { "mem": { - "heap_committed_in_bytes": 1073741824, - "heap_used_percent": 14, - "heap_max_in_bytes": 1073741822, - "non_heap_committed_in_bytes": 161742848, - "heap_used_in_bytes": 151451136, - "non_heap_used_in_bytes": 152187584 + "heap_committed_in_bytes": 264241152, + "heap_used_percent": 2, + "heap_max_in_bytes": 5184159742, + "non_heap_committed_in_bytes": 191889408, + "heap_used_in_bytes": 143564464, + "non_heap_used_in_bytes": 180940656 }, "threads": { - "count": 44, - "peak_count": 50 - } + "count": 83, + "peak_count": 85 + }, + "uptime_in_millis": 448206 }, "logstash": { "pipeline": { "batch_delay": 50, "batch_size": 125, - "workers": 16 + "workers": 8 }, "pipelines": [ - "sink", - "source" + "standalone-pipeline", + "pipeline-with-memory-queue", + "pipeline-with-persisted-queue" ], - "http_address": "127.0.0.1:9600", - "host": "MacBook-Pro.local", - "name": "rb", - "ephemeral_id": "41b5aa37-2e35-4054-bad6-56b517c22b00", - "uuid": "b924ad43-2dbe-42b4-8931-75a52aa7fcbf", - "version": "8.8.2", + "http_address": "0.0.0.0:9600", + "name": "21d61ee7529e", + "host": "21d61ee7529e", + "ephemeral_id": "fa27552b-e31d-463d-a5db-f470e6c2f0ba", + "version": "8.6.0", + "uuid": "2566e68f-ea0e-4dd0-8b65-17bc7bd9f685", "snapshot": false, "status": "green" }, "process": { + "open_file_descriptors": 94, "mem": { - "total_virtual_in_bytes": 40270282752 + "total_virtual_in_bytes": 11442712576 }, - "open_file_descriptors": 101, - "max_file_descriptors": 10240, + "max_file_descriptors": 1048576, "cpu": { "load_average": { - "1m": 4.91650390625 + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 }, - "total_in_millis": 61419, - "percent": 0 + "total_in_millis": 130690, + "percent": 2 }, - "peak_open_file_descriptors": 105 + "peak_open_file_descriptors": 95 }, "os": { "cpu": { "load_average": { - "1m": 4.91650390625 + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 }, - "total_in_millis": 61419, - "percent": 0 + "total_in_millis": 130690, + "percent": 2 }, "cgroup": {} }, "events": { - "filtered": 0, - "in": 0, - "queue_push_duration_in_millis": 0, - "duration_in_millis": 61, - "out": 0 + "filtered": 27752, + "in": 28442, + "queue_push_duration_in_millis": 597, + "duration_in_millis": 3202220, + "out": 27752 }, "queue": { "events_count": 0 @@ -82,12 +88,12 @@ }, "agent": { "name": "MacBook-Pro.local", - "id": "22c7007f-bdf3-4aaf-97bd-b01a5e9951e1", - "ephemeral_id": "966b3a32-dec5-41ab-94e4-aba8868c29b1", + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", "type": "filebeat", + "ephemeral_id": "e24a6e70-8e93-4d18-8535-319e63c81bc8", "version": "8.10.1" }, - "@timestamp": "2023-10-03T20:59:28.221Z", + "@timestamp": "2023-10-04T18:53:48.769Z", "ecs": { "version": "8.0.0" }, @@ -96,14 +102,19 @@ "type": "metrics", "dataset": "logstash.node" }, + "elastic_agent": { + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", + "version": "8.10.1", + "snapshot": false + }, "host": { "hostname": "macbook-pro.local", "os": { "build": "22F82", "kernel": "22.5.0", "name": "macOS", - "type": "macos", "family": "darwin", + "type": "macos", "version": "13.4.1", "platform": "darwin" }, @@ -113,18 +124,13 @@ "name": "macbook-pro.local", "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", "mac": [ - "82-C1-07-06-9C-00" + "AC-DE-48-00-11-22" ], "architecture": "x86_64" }, - "elastic_agent": { - "id": "22c7007f-bdf3-4aaf-97bd-b01a5e9951e1", - "version": "8.10.1", - "snapshot": false - }, "event": { "agent_id_status": "verified", - "ingested": "2023-10-03T20:59:30Z", + "ingested": "2023-10-04T18:53:49Z", "dataset": "logstash.node" } } \ No newline at end of file diff --git a/packages/logstash/data_stream/pipeline/fields/agent.yml b/packages/logstash/data_stream/pipeline/fields/agent.yml new file mode 100644 index 00000000000..d0032fb9fff --- /dev/null +++ b/packages/logstash/data_stream/pipeline/fields/agent.yml @@ -0,0 +1,141 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + - name: availability_zone + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + - name: instance.id + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + - name: instance.name + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + - name: provider + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + - name: region + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + type: object + object_type: keyword + description: Image labels. + - name: name + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + type: keyword + ignore_above: 1024 + description: Operating system architecture. + - name: domain + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + default_field: false + - name: hostname + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + type: ip + description: Host ip addresses. + - name: mac + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + - name: os.kernel + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + - name: os.name + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + - name: os.platform + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + - name: os.version + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + - name: type + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + description: > + OS build information. + + - name: os.codename + type: keyword + description: > + OS codename, if any. + diff --git a/packages/logstash/data_stream/pipeline/sample_event.json b/packages/logstash/data_stream/pipeline/sample_event.json index af0fd83ab54..c3e8687a1ec 100644 --- a/packages/logstash/data_stream/pipeline/sample_event.json +++ b/packages/logstash/data_stream/pipeline/sample_event.json @@ -1,114 +1,100 @@ { + "@timestamp": "2023-10-04T18:53:18.708Z", + "data_stream": { + "dataset": "logstash.pipeline", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.pipeline", + "ingested": "2023-10-04T18:53:19Z" + }, + "host": { + "architecture": "x86_64", + "hostname": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "ip": [ + "192.168.1.184" + ], + "mac": [ + "AC-DE-48-00-11-22" + ], + "name": "macbook-pro.local", + "os": { + "build": "22F82", + "family": "darwin", + "kernel": "22.5.0", + "name": "macOS", + "platform": "darwin", + "version": "13.4.1" + } + }, + "input": { + "type": "cel" + }, "logstash": { "pipeline": { - "elasticsearch.cluster.id": [], + "host": { + "address": "0.0.0.0:9600", + "name": "21d61ee7529e" + }, + "name": "standalone-pipeline", "total": { - "queues": { - "current_size": { - "bytes": 1 - }, - "type": "persisted", - "max_size": { - "bytes": 1073741824 - }, - "events": 0 - }, - "time": { - "duration": { - "ms": 2 - }, - "queue_push_duration": { - "ms": 0 - } + "events": { + "filtered": 2038, + "in": 2038, + "out": 2038 }, "flow": { - "worker_concurrency": { - "current": 0.00009574 - }, - "queue_backpressure": { - "current": 0 + "filter_throughput": { + "current": 5.02, + "last_1_minute": 5.003 }, "input_throughput": { - "current": 0 - }, - "queue_persisted_growth_events": { - "current": 0 + "current": 4.948, + "last_1_minute": 5.003 }, - "queue_persisted_growth_bytes": { - "current": 0 + "output_throughput": { + "current": 5.02, + "last_1_minute": 5.003 }, - "filter_throughput": { - "current": 0 + "queue_backpressure": { + "current": 0, + "last_1_minute": 0 }, - "output_throughput": { - "current": 0 + "worker_concurrency": { + "current": 0.001, + "last_1_minute": 0.001 } }, - "events": { - "filtered": 0, - "in": 0, - "out": 0 + "queues": { + "current_size": { + "bytes": 0 + }, + "events": 0, + "max_size": { + "bytes": 0 + }, + "type": "memory" }, "reloads": { "failures": 0, "successes": 0 + }, + "time": { + "duration": { + "ms": 1363 + }, + "queue_push_duration": { + "ms": 12 + } } - }, - "name": "source", - "host": { - "address": "127.0.0.1:9600", - "name": "rb" } } - }, - "input": { - "type": "cel" - }, - "agent": { - "name": "MacBook-Pro.local", - "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", - "ephemeral_id": "1fea7ffc-92ee-4a81-9402-10ef28b471f8", - "type": "filebeat", - "version": "8.10.1" - }, - "@timestamp": "2023-10-03T20:37:08.677Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.pipeline" - }, - "host": { - "hostname": "macbook-pro.local", - "os": { - "build": "22F82", - "kernel": "22.5.0", - "name": "macOS", - "family": "darwin", - "type": "macos", - "version": "13.4.1", - "platform": "darwin" - }, - "ip": [ - "192.168.1.184" - ], - "name": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "mac": [ - "82-C1-07-06-9C-00" - ], - "architecture": "x86_64" - }, - "elastic_agent": { - "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", - "version": "8.10.1", - "snapshot": false - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-03T20:37:09Z", - "dataset": "logstash.pipeline" } -} \ No newline at end of file +} + diff --git a/packages/logstash/data_stream/plugins/fields/agent.yml b/packages/logstash/data_stream/plugins/fields/agent.yml new file mode 100644 index 00000000000..d0032fb9fff --- /dev/null +++ b/packages/logstash/data_stream/plugins/fields/agent.yml @@ -0,0 +1,141 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + - name: availability_zone + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + - name: instance.id + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + - name: instance.name + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + - name: provider + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + - name: region + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + type: object + object_type: keyword + description: Image labels. + - name: name + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + type: keyword + ignore_above: 1024 + description: Operating system architecture. + - name: domain + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + default_field: false + - name: hostname + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + type: ip + description: Host ip addresses. + - name: mac + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + - name: os.kernel + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + - name: os.name + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + - name: os.platform + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + - name: os.version + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + - name: type + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + description: > + OS build information. + + - name: os.codename + type: keyword + description: > + OS codename, if any. + diff --git a/packages/logstash/data_stream/plugins/sample_event.json b/packages/logstash/data_stream/plugins/sample_event.json index 8846517955d..6507f0d50ea 100644 --- a/packages/logstash/data_stream/plugins/sample_event.json +++ b/packages/logstash/data_stream/plugins/sample_event.json @@ -1,86 +1,74 @@ { - "logstash": { - "pipeline": { - "elasticsearch.cluster.id": [ - "_W1QSt-NRWGRG6nNFYxxZg", - "9MOGoKiESvaklNVmxLo3iA" - ], - "plugin": { - "input": { - "name": "pipeline", - "time": { - "queue_push_duration": { - "ms": 0 - } - }, - "id": "a30c4938ad8deb314752397cfe6f356b0441dfb67bf10f3ebaad455596b164e6", - "flow": { - "throughput": { - "current": 0 - } - }, - "events": { - "out": 0 - } - }, - "type": "input" - }, - "name": "sink", - "host": { - "address": "127.0.0.1:9600", - "name": "rb" - }, - "id": "81e8debdd29bc61f6416dcb7d305f8da144f2e15ae10a6d37d75d231100c8441" - } - }, - "input": { - "type": "cel" - }, - "agent": { - "name": "MacBook-Pro.local", - "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", - "ephemeral_id": "1fea7ffc-92ee-4a81-9402-10ef28b471f8", - "type": "filebeat", - "version": "8.10.1" + "@timestamp": "2023-10-04T18:51:48.661Z", + "data_stream": { + "dataset": "logstash.plugins", + "namespace": "default", + "type": "metrics" }, - "@timestamp": "2023-10-03T20:37:08.739Z", "ecs": { "version": "8.0.0" }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.plugins" - }, - "elastic_agent": { - "id": "b5a15cae-c060-42a7-981a-d833c94f0a6b", - "version": "8.10.1", - "snapshot": false + "event": { + "agent_id_status": "verified", + "dataset": "logstash.plugins", + "ingested": "2023-10-04T18:51:49Z" }, "host": { + "architecture": "x86_64", "hostname": "macbook-pro.local", - "os": { - "build": "22F82", - "kernel": "22.5.0", - "name": "macOS", - "type": "macos", - "family": "darwin", - "version": "13.4.1", - "platform": "darwin" - }, + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", "ip": [ "192.168.1.184" ], - "name": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", "mac": [ - "82-C1-07-06-9C-00" + "AC-DE-48-00-11-22" ], - "architecture": "x86_64" + "name": "macbook-pro.local", + "os": { + "build": "22F82", + "family": "darwin", + "kernel": "22.5.0", + "name": "macOS", + "platform": "darwin", + "version": "13.4.1" + } }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-03T20:37:09Z", - "dataset": "logstash.plugins" + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "elasticsearch": { + "cluster": { + "id": "2IcePuo6RCi_511abdJxLQ" + } + }, + "host": { + "address": "0.0.0.0:9600", + "name": "21d61ee7529e" + }, + "id": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", + "name": "pipeline-with-memory-queue", + "plugin": { + "codec": { + "decode": { + "duration": { + "ms": 0 + }, + "in": 0, + "out": 0 + }, + "encode": { + "duration": { + "ms": 0 + }, + "in": 0 + }, + "id": "plain_f62ef0fc-3db5-44fb-a464-0458e6181aa0", + "name": "plain" + }, + "type": "codec" + } + } } } \ No newline at end of file diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 5c52e7e8355..32593b3cba8 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -6,6 +6,15 @@ The `logstash` package collects metrics and logs of Logstash. The `logstash` package works with Logstash 8.5.0 and later +## Technical Preview note + +This Logstash package also includes a technical preview of Logstash data collection and dashboards +native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. + +Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, +and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` + + ## Logs Logstash package supports the plain text format and the JSON format. Also, two types of @@ -528,3 +537,665 @@ An example event for `node` looks as following: } } ``` + + +## Metrics (Technical Preview) + +This Logstash package also includes a technical preview of Logstash data collection and dashboards +native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. + +Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, +and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` + +### Fields and Sample Event + +#### Node + +This is the `node` dataset, which drives the Node dashboard pages. + +#### Example + +**Exported fields** + +| Field | Description | Type | Metric Type | +|---|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | +| cloud.instance.name | Instance name of the host machine. | keyword | | +| cloud.machine.type | Machine type of the host machine. | keyword | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | +| cloud.region | Region in which this host is running. | keyword | | +| cluster_uuid | | alias | | +| container.id | Unique container id. | keyword | | +| container.image.name | Name of the image the container was built on. | keyword | | +| container.labels | Image labels. | object | | +| container.name | Container name. | keyword | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | +| error.message | Error message. | match_only_text | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | +| host.architecture | Operating system architecture. | keyword | | +| host.containerized | If the host is a container. | boolean | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.os.build | OS build information. | keyword | | +| host.os.codename | OS codename, if any. | keyword | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | +| host.os.name | Operating system name, without the version. | keyword | | +| host.os.name.text | Multi-field of `host.os.name`. | text | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | +| host.os.version | Operating system version as a raw string. | keyword | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | +| input.type | | keyword | | +| logstash.elasticsearch.cluster.id | | keyword | | +| logstash.host.address | | alias | | +| logstash.host.name | | alias | | +| logstash.node.stats.events.duration_in_millis | | long | counter | +| logstash.node.stats.events.filtered | Filtered events counter | long | counter | +| logstash.node.stats.events.in | Incoming events counter | long | counter | +| logstash.node.stats.events.out | Outgoing events counter | long | counter | +| logstash.node.stats.events.queue_push_duration_in_millis | | long | counter | +| logstash.node.stats.jvm.gc.collectors.old.collection_count | | long | counter | +| logstash.node.stats.jvm.gc.collectors.old.collection_time_in_millis | | long | counter | +| logstash.node.stats.jvm.gc.collectors.young.collection_count | | long | counter | +| logstash.node.stats.jvm.gc.collectors.young.collection_time_in_millis | | long | counter | +| logstash.node.stats.jvm.mem.heap_committed_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | counter | +| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.heap_used_percent | | long | gauge | +| logstash.node.stats.jvm.mem.non_heap_committed_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.non_heap_used_in_bytes | | long | gauge | +| logstash.node.stats.jvm.threads.count | current number of threads | long | counter | +| logstash.node.stats.jvm.threads.peak_count | peak number of threads | long | counter | +| logstash.node.stats.jvm.uptime_in_millis | | long | counter | +| logstash.node.stats.logstash.ephemeral_id | | keyword | | +| logstash.node.stats.logstash.host | | keyword | | +| logstash.node.stats.logstash.http_address | | keyword | | +| logstash.node.stats.logstash.name | | keyword | | +| logstash.node.stats.logstash.pipeline.batch_delay | | long | gauge | +| logstash.node.stats.logstash.pipeline.batch_size | | long | gauge | +| logstash.node.stats.logstash.pipeline.workers | | long | gauge | +| logstash.node.stats.logstash.pipelines | | keyword | | +| logstash.node.stats.logstash.snapshot | | boolean | | +| logstash.node.stats.logstash.status | | keyword | | +| logstash.node.stats.logstash.uuid | | keyword | | +| logstash.node.stats.logstash.version | | keyword | | +| logstash.node.stats.os.cgroup.cpu.cfs_quota_micros | | long | gauge | +| logstash.node.stats.os.cgroup.cpu.control_group | | keyword | | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | gauge | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | counter | +| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | counter | +| logstash.node.stats.os.cgroup.cpuacct.control_group | | keyword | | +| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | counter | +| logstash.node.stats.os.cpu.load_average.15m | | half_float | gauge | +| logstash.node.stats.os.cpu.load_average.1m | | half_float | gauge | +| logstash.node.stats.os.cpu.load_average.5m | | half_float | gauge | +| logstash.node.stats.os.cpu.percent | | double | gauge | +| logstash.node.stats.os.cpu.total_in_millis | | long | counter | +| logstash.node.stats.pipelines.ephemeral_id | | keyword | | +| logstash.node.stats.pipelines.events.duration_in_millis | | long | | +| logstash.node.stats.pipelines.events.filtered | | long | | +| logstash.node.stats.pipelines.events.in | | long | | +| logstash.node.stats.pipelines.events.out | | long | | +| logstash.node.stats.pipelines.events.queue_push_duration_in_millis | | long | | +| logstash.node.stats.pipelines.hash | | keyword | | +| logstash.node.stats.pipelines.id | | keyword | | +| logstash.node.stats.pipelines.queue.events_count | | long | | +| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | | +| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | | +| logstash.node.stats.pipelines.queue.type | | keyword | | +| logstash.node.stats.pipelines.reloads.failures | | long | | +| logstash.node.stats.pipelines.reloads.successes | | long | | +| logstash.node.stats.process.cpu.load_average.15m | | half_float | gauge | +| logstash.node.stats.process.cpu.load_average.1m | | half_float | gauge | +| logstash.node.stats.process.cpu.load_average.5m | | half_float | gauge | +| logstash.node.stats.process.cpu.percent | | double | gauge | +| logstash.node.stats.process.cpu.total_in_millis | | long | counter | +| logstash.node.stats.process.max_file_descriptors | | long | gauge | +| logstash.node.stats.process.mem.total_virtual_in_bytes | | long | gauge | +| logstash.node.stats.process.open_file_descriptors | | long | gauge | +| logstash.node.stats.process.peak_open_file_descriptors | | long | gauge | +| logstash.node.stats.queue.events_count | | long | counter | +| logstash.node.stats.reloads.failures | | long | counter | +| logstash.node.stats.reloads.successes | | long | counter | +| logstash.node.stats.timestamp | | date | | +| logstash.pipeline.name | | alias | | +| process.pid | Process id. | long | | +| service.address | Address where data about this service was collected from. | keyword | | +| service.hostname | Hostname of the service | keyword | | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | + + +An example event for `node_cel` looks as following: + +```json +{ + "logstash": { + "node": { + "stats": { + "jvm": { + "mem": { + "heap_committed_in_bytes": 264241152, + "heap_used_percent": 2, + "heap_max_in_bytes": 5184159742, + "non_heap_committed_in_bytes": 191889408, + "heap_used_in_bytes": 143564464, + "non_heap_used_in_bytes": 180940656 + }, + "threads": { + "count": 83, + "peak_count": 85 + }, + "uptime_in_millis": 448206 + }, + "logstash": { + "pipeline": { + "batch_delay": 50, + "batch_size": 125, + "workers": 8 + }, + "pipelines": [ + "standalone-pipeline", + "pipeline-with-memory-queue", + "pipeline-with-persisted-queue" + ], + "http_address": "0.0.0.0:9600", + "name": "21d61ee7529e", + "host": "21d61ee7529e", + "ephemeral_id": "fa27552b-e31d-463d-a5db-f470e6c2f0ba", + "version": "8.6.0", + "uuid": "2566e68f-ea0e-4dd0-8b65-17bc7bd9f685", + "snapshot": false, + "status": "green" + }, + "process": { + "open_file_descriptors": 94, + "mem": { + "total_virtual_in_bytes": 11442712576 + }, + "max_file_descriptors": 1048576, + "cpu": { + "load_average": { + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 + }, + "total_in_millis": 130690, + "percent": 2 + }, + "peak_open_file_descriptors": 95 + }, + "os": { + "cpu": { + "load_average": { + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 + }, + "total_in_millis": 130690, + "percent": 2 + }, + "cgroup": {} + }, + "events": { + "filtered": 27752, + "in": 28442, + "queue_push_duration_in_millis": 597, + "duration_in_millis": 3202220, + "out": 27752 + }, + "queue": { + "events_count": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + } + } + } + }, + "input": { + "type": "cel" + }, + "agent": { + "name": "MacBook-Pro.local", + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", + "type": "filebeat", + "ephemeral_id": "e24a6e70-8e93-4d18-8535-319e63c81bc8", + "version": "8.10.1" + }, + "@timestamp": "2023-10-04T18:53:48.769Z", + "ecs": { + "version": "8.0.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.node" + }, + "elastic_agent": { + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", + "version": "8.10.1", + "snapshot": false + }, + "host": { + "hostname": "macbook-pro.local", + "os": { + "build": "22F82", + "kernel": "22.5.0", + "name": "macOS", + "family": "darwin", + "type": "macos", + "version": "13.4.1", + "platform": "darwin" + }, + "ip": [ + "192.168.1.184" + ], + "name": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "AC-DE-48-00-11-22" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-04T18:53:49Z", + "dataset": "logstash.node" + } +} +``` + +#### Pipeline + +This is the `pipeline` dataset, which drives the Pipeline dashboard pages. + +#### Example + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | | +| cloud.instance.name | Instance name of the host machine. | keyword | | | +| cloud.machine.type | Machine type of the host machine. | keyword | | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | +| cloud.region | Region in which this host is running. | keyword | | | +| cluster_uuid | | alias | | | +| container.id | Unique container id. | keyword | | | +| container.image.name | Name of the image the container was built on. | keyword | | | +| container.labels | Image labels. | object | | | +| container.name | Container name. | keyword | | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | +| host.architecture | Operating system architecture. | keyword | | | +| host.containerized | If the host is a container. | boolean | | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | +| host.ip | Host ip addresses. | ip | | | +| host.mac | Host mac addresses. | keyword | | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | +| host.os.build | OS build information. | keyword | | | +| host.os.codename | OS codename, if any. | keyword | | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | +| host.os.name | Operating system name, without the version. | keyword | | | +| host.os.name.text | Multi-field of `host.os.name`. | text | | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | +| host.os.version | Operating system version as a raw string. | keyword | | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | +| input.type | | keyword | | | +| logstash.host.address | | alias | | | +| logstash.host.name | | alias | | | +| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | +| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | +| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | +| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | +| logstash.pipeline.total.events.filtered | Number of events filtered by the pipeline | long | | counter | +| logstash.pipeline.total.events.in | Number of events received by the pipeline | long | | counter | +| logstash.pipeline.total.events.out | Number of events emitted by the pipeline | long | | counter | +| logstash.pipeline.total.flow.filter_throughput.current | current value of the filter throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.filter_throughput.last_1_minute | current value of the filter throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.input_throughput.current | current value of the input throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.input_throughput.last_1_minute | current value of the throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.output_throughput.current | current value of the output throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.output_throughput.last_1_minute | current value of the output throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_backpressure.current | current value of the queue backpressure flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_backpressure.last_1_minute | current value of the queue backpressure flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_bytes.current | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_minute | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_events.current | current value of the queue persisted growth events flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_minute | current value of the queue persisted growth events flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_concurrency.current | current value of the worker concurrency flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_concurrency.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | +| logstash.pipeline.total.queues.current_size.bytes | Current size of the PQ | long | byte | gauge | +| logstash.pipeline.total.queues.events | Number of events in the PQ for this pipeline | long | | counter | +| logstash.pipeline.total.queues.max_size.bytes | Maximum possible size of the PQ | long | | gauge | +| logstash.pipeline.total.queues.type | Type of queue - persistent or memory | keyword | | | +| logstash.pipeline.total.reloads.failures | Number of failed reloads for this pipeline | long | | counter | +| logstash.pipeline.total.reloads.successes | Number of successful reloads for this pipeline | long | | counter | +| logstash.pipeline.total.time.duration.ms | Time spent processing events through the pipeline. | long | ms | counter | +| logstash.pipeline.total.time.queue_push_duration.ms | Time spent pushing events to the queue for this pipeline. | long | ms | counter | +| process.pid | Process id. | long | | | +| service.address | Address where data about this service was collected from. | keyword | | | +| service.hostname | Hostname of the service | keyword | | | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | + + +An example event for `pipeline` looks as following: + +```json +{ + "@timestamp": "2023-10-04T18:53:18.708Z", + "data_stream": { + "dataset": "logstash.pipeline", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.pipeline", + "ingested": "2023-10-04T18:53:19Z" + }, + "host": { + "architecture": "x86_64", + "hostname": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "ip": [ + "192.168.1.184" + ], + "mac": [ + "AC-DE-48-00-11-22" + ], + "name": "macbook-pro.local", + "os": { + "build": "22F82", + "family": "darwin", + "kernel": "22.5.0", + "name": "macOS", + "platform": "darwin", + "version": "13.4.1" + } + }, + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "host": { + "address": "0.0.0.0:9600", + "name": "21d61ee7529e" + }, + "name": "standalone-pipeline", + "total": { + "events": { + "filtered": 2038, + "in": 2038, + "out": 2038 + }, + "flow": { + "filter_throughput": { + "current": 5.02, + "last_1_minute": 5.003 + }, + "input_throughput": { + "current": 4.948, + "last_1_minute": 5.003 + }, + "output_throughput": { + "current": 5.02, + "last_1_minute": 5.003 + }, + "queue_backpressure": { + "current": 0, + "last_1_minute": 0 + }, + "worker_concurrency": { + "current": 0.001, + "last_1_minute": 0.001 + } + }, + "queues": { + "current_size": { + "bytes": 0 + }, + "events": 0, + "max_size": { + "bytes": 0 + }, + "type": "memory" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "time": { + "duration": { + "ms": 1363 + }, + "queue_push_duration": { + "ms": 12 + } + } + } + } + } +} +``` + +#### Plugin + +This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the +pipeline collection period, and setting it to an appropriate value. + +#### Example + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | | +| cloud.instance.name | Instance name of the host machine. | keyword | | | +| cloud.machine.type | Machine type of the host machine. | keyword | | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | +| cloud.region | Region in which this host is running. | keyword | | | +| cluster_uuid | | alias | | | +| container.id | Unique container id. | keyword | | | +| container.image.name | Name of the image the container was built on. | keyword | | | +| container.labels | Image labels. | object | | | +| container.name | Container name. | keyword | | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | +| host.architecture | Operating system architecture. | keyword | | | +| host.containerized | If the host is a container. | boolean | | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | +| host.ip | Host ip addresses. | ip | | | +| host.mac | Host mac addresses. | keyword | | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | +| host.os.build | OS build information. | keyword | | | +| host.os.codename | OS codename, if any. | keyword | | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | +| host.os.name | Operating system name, without the version. | keyword | | | +| host.os.name.text | Multi-field of `host.os.name`. | text | | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | +| host.os.version | Operating system version as a raw string. | keyword | | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | +| input.type | | keyword | | | +| logstash.host.address | | alias | | | +| logstash.host.name | | alias | | | +| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | +| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | +| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | +| logstash.pipeline.id | Logstash Pipeline hash | keyword | | | +| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | +| logstash.pipeline.plugin.codec.decode.duration.ms | amount of time spend decoding events | long | ms | counter | +| logstash.pipeline.plugin.codec.decode.in | number of events entering the decoder | long | | counter | +| logstash.pipeline.plugin.codec.decode.out | number of events exiting the decoder | long | | counter | +| logstash.pipeline.plugin.codec.encode.duration.ms | amount of time spend encoding events | long | ms | counter | +| logstash.pipeline.plugin.codec.encode.in | number of events encoded | long | | counter | +| logstash.pipeline.plugin.codec.id | Id of codec plugin | keyword | | | +| logstash.pipeline.plugin.codec.name | Name of codec plugin | keyword | | | +| logstash.pipeline.plugin.filter.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.filter.events.in | number of events received by the filter | long | | counter | +| logstash.pipeline.plugin.filter.events.out | number of events emitted by the filter | long | | counter | +| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.id | Id of filter plugin | keyword | | | +| logstash.pipeline.plugin.filter.name | Name of filter plugin | keyword | | | +| logstash.pipeline.plugin.filter.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | +| logstash.pipeline.plugin.input.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.input.events.out | number of events emitted by the input | long | | counter | +| logstash.pipeline.plugin.input.flow.throughput.current | throughput of this input plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.input.flow.throughput.last_1_minute | throughput of this input plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.input.id | Id of input plugin | keyword | | | +| logstash.pipeline.plugin.input.name | Name of input plugin | keyword | | | +| logstash.pipeline.plugin.input.time.queue_push_duration.ms | amount of time spend pushing events to the queue | long | ms | counter | +| logstash.pipeline.plugin.output.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.output.events.in | number of events received by the output | long | | counter | +| logstash.pipeline.plugin.output.events.out | number of events emitted by the output | long | | counter | +| logstash.pipeline.plugin.output.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.id | Id of output plugin | keyword | | | +| logstash.pipeline.plugin.output.name | Name of output plugin | keyword | | | +| logstash.pipeline.plugin.output.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | +| logstash.pipeline.plugin.type | Type of the plugin | keyword | | | +| process.pid | Process id. | long | | | +| service.address | Address where data about this service was collected from. | keyword | | | +| service.hostname | Hostname of the service | keyword | | | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | + + +An example event for `plugins` looks as following: + +```json +{ + "@timestamp": "2023-10-04T18:51:48.661Z", + "data_stream": { + "dataset": "logstash.plugins", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.plugins", + "ingested": "2023-10-04T18:51:49Z" + }, + "host": { + "architecture": "x86_64", + "hostname": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "ip": [ + "192.168.1.184" + ], + "mac": [ + "AC-DE-48-00-11-22" + ], + "name": "macbook-pro.local", + "os": { + "build": "22F82", + "family": "darwin", + "kernel": "22.5.0", + "name": "macOS", + "platform": "darwin", + "version": "13.4.1" + } + }, + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "elasticsearch": { + "cluster": { + "id": "2IcePuo6RCi_511abdJxLQ" + } + }, + "host": { + "address": "0.0.0.0:9600", + "name": "21d61ee7529e" + }, + "id": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", + "name": "pipeline-with-memory-queue", + "plugin": { + "codec": { + "decode": { + "duration": { + "ms": 0 + }, + "in": 0, + "out": 0 + }, + "encode": { + "duration": { + "ms": 0 + }, + "in": 0 + }, + "id": "plain_f62ef0fc-3db5-44fb-a464-0458e6181aa0", + "name": "plain" + }, + "type": "codec" + } + } + } +} +``` diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index 6e6ab181ade..f9a73814847 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1156,7 +1156,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:22:03.327Z", + "created_at": "2023-10-04T16:10:54.914Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index 2da42cfc2df..79c148f4683 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -1570,9 +1570,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:30:26.635Z", + "created_at": "2023-10-04T16:10:54.914Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index 6fced6e6c12..eca5ef5040d 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -1897,9 +1897,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:31:06.493Z", + "created_at": "2023-10-04T16:10:54.914Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index b3de737478e..974ab0a270f 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -407,6 +407,257 @@ "type": "lens", "version": "8.10.1" }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "4d13c1a7-6b92-47fe-8baf-7ca505bd9fda": { + "columnOrder": [ + "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3", + "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6" + ], + "columns": { + "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Average time processing per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "isFormulaBroken": false + }, + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.time.duration.ms" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "counter_rate", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "cumulative_sum", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "counter_rate", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "cumulative_sum", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5" + ], + "location": { + "max": 147, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "type": "function" + } + }, + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5" + ], + "scale": "ratio" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf" + ], + "layerId": "4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3", + "xAccessor": "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "Average Time processed per event (ms)", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "7d85767a-5dbb-4704-8716-1d4c30a5f675", + "w": 40, + "x": 8, + "y": 5 + }, + "panelIndex": "7d85767a-5dbb-4704-8716-1d4c30a5f675", + "title": "Average Time processed per event (ms)", + "type": "lens", + "version": "8.10.1" + }, { "embeddableConfig": { "attributes": { @@ -563,7 +814,7 @@ "i": "f03395de-a163-4204-ac82-83c0ea542eeb", "w": 20, "x": 8, - "y": 5 + "y": 17 }, "panelIndex": "f03395de-a163-4204-ac82-83c0ea542eeb", "title": "Events Received Rate", @@ -726,7 +977,7 @@ "i": "4fad1f03-99d3-4834-a6c4-bea283178567", "w": 20, "x": 28, - "y": 5 + "y": 17 }, "panelIndex": "4fad1f03-99d3-4834-a6c4-bea283178567", "title": "Events Emitted Rate", @@ -894,7 +1145,7 @@ "i": "a3444242-6e3e-4959-8815-43e65b3e8d69", "w": 20, "x": 8, - "y": 14 + "y": 26 }, "panelIndex": "a3444242-6e3e-4959-8815-43e65b3e8d69", "title": "Persistent Queue Size (events)", @@ -1137,7 +1388,7 @@ "i": "5171ef08-d301-4a9c-832b-40f41c9ad24d", "w": 20, "x": 28, - "y": 14 + "y": 26 }, "panelIndex": "5171ef08-d301-4a9c-832b-40f41c9ad24d", "title": "Persistent Queue Utilization (%)", @@ -1174,7 +1425,7 @@ "i": "681d5f97-e2f4-4a74-9710-7e75bb73fea8", "w": 24, "x": 16, - "y": 23 + "y": 35 }, "panelIndex": "681d5f97-e2f4-4a74-9710-7e75bb73fea8", "title": "Plugins", @@ -1188,7 +1439,7 @@ "references": [ { "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", "type": "index-pattern" } ], @@ -1197,54 +1448,268 @@ "datasourceStates": { "formBased": { "layers": { - "df79554f-08e3-4f7f-b438-80bc81e14637": { + "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { "columnOrder": [ - "65ca78d2-d895-4db6-bec3-7e110bd0030b", - "245e3126-537a-4644-b6b3-9a33800dfb91", - "0a5be802-a381-4edd-ad8c-488a6a419619", - "410757d6-0433-4cdd-b51b-138704dfc00a", - "d172976a-5c43-46b7-9d96-44ac40134cbe", - "3bdc4103-5313-4e00-a407-de5b7fde820c", - "a175cdde-215b-441e-84d6-ceab03b1495d", - "d172976a-5c43-46b7-9d96-44ac40134cbeX0", - "d172976a-5c43-46b7-9d96-44ac40134cbeX1", - "3bdc4103-5313-4e00-a407-de5b7fde820cX0", - "3bdc4103-5313-4e00-a407-de5b7fde820cX1" + "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "b02fb0ac-3ead-4546-ac07-2d21f54628dc", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" ], "columns": { - "0a5be802-a381-4edd-ad8c-488a6a419619": { - "customLabel": true, - "dataType": "string", + "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { + "dataType": "date", "isBucketed": true, - "label": "Plugin Id", - "operationType": "terms", + "label": "@timestamp", + "operationType": "date_histogram", "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 1000 + "dropPartials": false, + "includeEmptyRows": false, + "interval": "auto" }, - "scale": "ordinal", - "sourceField": "logstash.pipeline.plugin.input.id" + "scale": "interval", + "sourceField": "@timestamp" }, - "245e3126-537a-4644-b6b3-9a33800dfb91": { + "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { "customLabel": true, - "dataType": "string", - "isBucketed": true, - "label": "Host Name", - "operationType": "terms", - "params": { + "dataType": "number", + "isBucketed": false, + "label": "Worker Utilization", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + }, + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", + "isFormulaBroken": false + }, + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" + ], + "scale": "ratio" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + }, + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Worker Millis per event", + "operationType": "counter_rate", + "references": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "ff88ade5-20cd-4210-b34e-7712c5da3acb": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" + }, + "isBucketed": false, + "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.output.name", + "logstash.pipeline.plugin.output.id" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.host.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b02fb0ac-3ead-4546-ac07-2d21f54628dc" + ], + "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", + "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "03d92828-8549-4cd9-a7ea-12f25502cef1", + "w": 20, + "x": 28, + "y": 103 + }, + "panelIndex": "03d92828-8549-4cd9-a7ea-12f25502cef1", + "title": "Worker utilization", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "df79554f-08e3-4f7f-b438-80bc81e14637": { + "columnOrder": [ + "65ca78d2-d895-4db6-bec3-7e110bd0030b", + "245e3126-537a-4644-b6b3-9a33800dfb91", + "0a5be802-a381-4edd-ad8c-488a6a419619", + "410757d6-0433-4cdd-b51b-138704dfc00a", + "d172976a-5c43-46b7-9d96-44ac40134cbe", + "3bdc4103-5313-4e00-a407-de5b7fde820c", + "8db4855c-0536-4cf8-84bd-eb8af3b72829", + "a175cdde-215b-441e-84d6-ceab03b1495d", + "d172976a-5c43-46b7-9d96-44ac40134cbeX0", + "d172976a-5c43-46b7-9d96-44ac40134cbeX1", + "3bdc4103-5313-4e00-a407-de5b7fde820cX0", + "3bdc4103-5313-4e00-a407-de5b7fde820cX1", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X0", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X1", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X2", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X3", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X4", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X5", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X6" + ], + "columns": { + "0a5be802-a381-4edd-ad8c-488a6a419619": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Plugin Id", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.plugin.input.id" + }, + "245e3126-537a-4644-b6b3-9a33800dfb91": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host Name", + "operationType": "terms", + "params": { "exclude": [], "excludeIsRegex": false, "include": [], @@ -1268,13 +1733,13 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Time Spent Pushing to Queue", + "label": "Time spent pushing to queue", "operationType": "formula", "params": { "format": { "id": "number", "params": { - "decimals": 0, + "decimals": 2, "suffix": "ms" } }, @@ -1290,7 +1755,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "label": "Part of Time spent pushing to queue", "operationType": "max", "params": { "emptyAsNull": false @@ -1302,7 +1767,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms))", + "label": "Part of Time spent pushing to queue", "operationType": "counter_rate", "references": [ "3bdc4103-5313-4e00-a407-de5b7fde820cX0" @@ -1349,6 +1814,125 @@ "scale": "ordinal", "sourceField": "logstash.pipeline.plugin.input.name" }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent pushing per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "isFormulaBroken": false + }, + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X6" + ], + "scale": "ratio" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.time.queue_push_duration.ms" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "counter_rate", + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "cumulative_sum", + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X1" + ], + "scale": "ratio" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.plugin.input.events.out" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "counter_rate", + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "cumulative_sum", + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X4" + ], + "scale": "ratio" + }, + "8db4855c-0536-4cf8-84bd-eb8af3b72829X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X2", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X5" + ], + "location": { + "max": 170, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.time.queue_push_duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.input.events.out)))", + "type": "function" + } + }, + "references": [ + "8db4855c-0536-4cf8-84bd-eb8af3b72829X2", + "8db4855c-0536-4cf8-84bd-eb8af3b72829X5" + ], + "scale": "ratio" + }, "a175cdde-215b-441e-84d6-ceab03b1495d": { "customLabel": true, "dataType": "number", @@ -1357,9 +1941,15 @@ "query": "\"logstash.pipeline.plugin.input.flow.throughput.last_1_minute\": *" }, "isBucketed": false, - "label": "Throughput Flow", + "label": "Throughput flow", "operationType": "last_value", "params": { + "format": { + "id": "number", + "params": { + "decimals": 3 + } + }, "sortField": "@timestamp" }, "scale": "ratio", @@ -1369,7 +1959,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Events Emitted", + "label": "Events emitted", "operationType": "formula", "params": { "formula": "counter_rate(max(logstash.pipeline.plugin.input.events.out))", @@ -1384,7 +1974,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.events.out))", + "label": "Part of Events Emitted", "operationType": "max", "params": { "emptyAsNull": false @@ -1396,7 +1986,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of counter_rate(max(logstash.pipeline.plugin.input.events.out))", + "label": "Part of Events Emitted", "operationType": "counter_rate", "references": [ "d172976a-5c43-46b7-9d96-44ac40134cbeX0" @@ -1478,6 +2068,45 @@ "colorMode": "cell", "columnId": "3bdc4103-5313-4e00-a407-de5b7fde820c", "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "isTransposed": false, "palette": { "name": "positive", "params": { @@ -1507,44 +2136,50 @@ "type": "palette" } }, + { + "alignment": "center", + "columnId": "245e3126-537a-4644-b6b3-9a33800dfb91", + "isTransposed": false + }, { "alignment": "center", "colorMode": "cell", - "columnId": "a175cdde-215b-441e-84d6-ceab03b1495d", + "columnId": "8db4855c-0536-4cf8-84bd-eb8af3b72829", "isTransposed": false, "palette": { - "name": "positive", + "name": "status", "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "steps": 5, "stops": [ { - "color": "#d6e9e4", + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", "stop": 20 }, { - "color": "#aed3ca", + "color": "#d6bf57", "stop": 40 }, { - "color": "#85bdb1", + "color": "#e7664c", "stop": 60 }, { - "color": "#5aa898", + "color": "#cc5642", "stop": 80 - }, - { - "color": "#209280", - "stop": 100 } ] }, "type": "palette" } - }, - { - "alignment": "center", - "columnId": "245e3126-537a-4644-b6b3-9a33800dfb91", - "isTransposed": false } ], "layerId": "df79554f-08e3-4f7f-b438-80bc81e14637", @@ -1563,7 +2198,7 @@ "i": "9a34af7e-5f12-49b4-86e0-5577203711eb", "w": 40, "x": 8, - "y": 27 + "y": 39 }, "panelIndex": "9a34af7e-5f12-49b4-86e0-5577203711eb", "type": "lens", @@ -1767,7 +2402,7 @@ "i": "f94ee279-63d8-45de-bfa0-9c93416b9752", "w": 40, "x": 8, - "y": 34 + "y": 46 }, "panelIndex": "f94ee279-63d8-45de-bfa0-9c93416b9752", "title": "Time Spent Pushing to Queues", @@ -1976,7 +2611,7 @@ "i": "1eabaf75-95dd-4cae-95d5-f00efe978872", "w": 20, "x": 8, - "y": 43 + "y": 55 }, "panelIndex": "1eabaf75-95dd-4cae-95d5-f00efe978872", "title": "Events Emitted per second", @@ -2147,7 +2782,7 @@ "i": "3470dd76-7ebd-4b19-a3de-fa0082a72ee5", "w": 20, "x": 28, - "y": 43 + "y": 55 }, "panelIndex": "3470dd76-7ebd-4b19-a3de-fa0082a72ee5", "title": "Input Flow", @@ -2185,7 +2820,7 @@ "i": "7f5bcb0f-394c-4bbd-b17a-c163a28a9315", "w": 24, "x": 16, - "y": 52 + "y": 64 }, "panelIndex": "7f5bcb0f-394c-4bbd-b17a-c163a28a9315", "title": "Plugins", @@ -2363,17 +2998,17 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Events processed/ms", + "label": "Time spent filtering per event", "operationType": "formula", "params": { "format": { "id": "number", "params": { "decimals": 2, - "suffix": "/ms" + "suffix": "ms" } }, - "formula": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.time.duration.ms)))", + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.time.duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.events.in)))", "isFormulaBroken": false }, "references": [ @@ -2384,24 +3019,20 @@ "56e40393-65fa-403e-b757-05ae1fc04063X0": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.plugin.filter.events.in\": *" - }, "isBucketed": false, - "label": "Part of Events processed/ms", - "operationType": "last_value", + "label": "Part of Events filtered/ms", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.filter.events.in" + "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" }, "56e40393-65fa-403e-b757-05ae1fc04063X1": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events processed/ms", + "label": "Part of Events filtered/ms", "operationType": "counter_rate", "references": [ "56e40393-65fa-403e-b757-05ae1fc04063X0" @@ -2413,7 +3044,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events processed/ms", + "label": "Part of Events filtered/ms", "operationType": "cumulative_sum", "references": [ "56e40393-65fa-403e-b757-05ae1fc04063X1" @@ -2423,24 +3054,20 @@ "56e40393-65fa-403e-b757-05ae1fc04063X3": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.plugin.filter.time.duration.ms\": *" - }, "isBucketed": false, - "label": "Part of Events processed/ms", - "operationType": "last_value", + "label": "Part of Events filtered/ms", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.filter.time.duration.ms" + "sourceField": "logstash.pipeline.plugin.filter.events.in" }, "56e40393-65fa-403e-b757-05ae1fc04063X4": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events processed/ms", + "label": "Part of Events filtered/ms", "operationType": "counter_rate", "references": [ "56e40393-65fa-403e-b757-05ae1fc04063X3" @@ -2452,7 +3079,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events processed/ms", + "label": "Part of Events filtered/ms", "operationType": "cumulative_sum", "references": [ "56e40393-65fa-403e-b757-05ae1fc04063X4" @@ -2463,7 +3090,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events processed/ms", + "label": "Part of Events filtered/ms", "operationType": "math", "params": { "tinymathAst": { @@ -2472,11 +3099,11 @@ "56e40393-65fa-403e-b757-05ae1fc04063X5" ], "location": { - "max": 174, + "max": 160, "min": 0 }, "name": "divide", - "text": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.filter.time.duration.ms)))", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.time.duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.filter.events.in)))", "type": "function" } }, @@ -2907,11 +3534,11 @@ "params": { "colorStops": [ { - "color": "#cc5642", + "color": "#209280", "stop": 0 }, { - "color": "#e7664c", + "color": "#54b399", "stop": 20 }, { @@ -2919,11 +3546,11 @@ "stop": 40 }, { - "color": "#54b399", + "color": "#e7664c", "stop": 60 }, { - "color": "#209280", + "color": "#cc5642", "stop": 80 } ], @@ -2935,11 +3562,11 @@ "steps": 5, "stops": [ { - "color": "#cc5642", + "color": "#209280", "stop": 20 }, { - "color": "#e7664c", + "color": "#54b399", "stop": 40 }, { @@ -2947,11 +3574,11 @@ "stop": 60 }, { - "color": "#54b399", + "color": "#e7664c", "stop": 80 }, { - "color": "#209280", + "color": "#cc5642", "stop": 100 } ] @@ -2985,7 +3612,7 @@ "i": "97bcb728-f0e5-4047-aade-2ef56bd33c22", "w": 40, "x": 8, - "y": 56 + "y": 68 }, "panelIndex": "97bcb728-f0e5-4047-aade-2ef56bd33c22", "type": "lens", @@ -3177,219 +3804,10 @@ } }, "references": [ - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" - ], - "scale": "ratio" - }, - "d531162d-3ee2-4de1-8427-c57e8d679cde": { - "dataType": "date", - "isBucketed": true, - "label": "@timestamp", - "operationType": "date_histogram", - "params": { - "dropPartials": false, - "includeEmptyRows": true, - "interval": "auto" - }, - "scale": "interval", - "sourceField": "@timestamp" - } - }, - "incompleteColumns": {}, - "sampling": 1 - } - } - }, - "textBased": { - "layers": {} - } - }, - "filters": [], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "axisTitlesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "curveType": "CURVE_MONOTONE_X", - "fittingFunction": "Linear", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "b02fb0ac-3ead-4546-ac07-2d21f54628dc" - ], - "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", - "layerType": "data", - "palette": { - "name": "default", - "type": "palette" - }, - "seriesType": "line", - "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", - "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" - } - ], - "legend": { - "isVisible": true, - "position": "right" - }, - "preferredSeriesType": "line", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "hide" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 10, - "i": "f8033492-7353-4564-83d5-6715e387f9ac", - "w": 40, - "x": 8, - "y": 70 - }, - "panelIndex": "f8033492-7353-4564-83d5-6715e387f9ac", - "title": "Average time spent processing each event", - "type": "lens", - "version": "8.10.1" - }, - { - "embeddableConfig": { - "attributes": { - "description": "", - "references": [ - { - "id": "logstash-sm-metrics", - "name": "indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { - "columnOrder": [ - "d531162d-3ee2-4de1-8427-c57e8d679cde", - "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", - "b02fb0ac-3ead-4546-ac07-2d21f54628dc", - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" - ], - "columns": { - "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { - "dataType": "string", - "isBucketed": true, - "label": "Top values of logstash.host.name + 2 others", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderAgg": { - "dataType": "number", - "isBucketed": false, - "label": "Count of logstash.pipeline.total.events.out", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.total.events.out" - }, - "orderBy": { - "type": "custom" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "multi_terms" - }, - "secondaryFields": [ - "logstash.pipeline.plugin.filter.name", - "logstash.pipeline.plugin.filter.id" - ], - "size": 20 - }, - "scale": "ordinal", - "sourceField": "logstash.host.name" - }, - "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Events Received per second", - "operationType": "formula", - "params": { - "format": { - "id": "number", - "params": { - "decimals": 2 - } - }, - "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.in))", - "isFormulaBroken": false - }, - "references": [ - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" - ], - "scale": "ratio", - "timeScale": "s" - }, - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.plugin.filter.events.in\": *" - }, - "isBucketed": false, - "label": "Part of Events Emitted per second", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.filter.events.in" - }, - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Events Emitted per second", - "operationType": "counter_rate", - "references": [ - "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX2", + "b02fb0ac-3ead-4546-ac07-2d21f54628dcX5" ], - "scale": "ratio", - "timeScale": "s" + "scale": "ratio" }, "d531162d-3ee2-4de1-8427-c57e8d679cde": { "dataType": "date", @@ -3475,14 +3893,14 @@ "hidePanelTitles": false }, "gridData": { - "h": 11, - "i": "d6db9db8-7da0-435c-8118-9d0c36d2573d", - "w": 20, + "h": 10, + "i": "f8033492-7353-4564-83d5-6715e387f9ac", + "w": 40, "x": 8, - "y": 80 + "y": 82 }, - "panelIndex": "d6db9db8-7da0-435c-8118-9d0c36d2573d", - "title": "Events Received per second", + "panelIndex": "f8033492-7353-4564-83d5-6715e387f9ac", + "title": "Average time spent processing each event", "type": "lens", "version": "8.10.1" }, @@ -3554,7 +3972,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Events Emitted per second", + "label": "Events Received per second", "operationType": "formula", "params": { "format": { @@ -3563,7 +3981,7 @@ "decimals": 2 } }, - "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.out))", + "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.in))", "isFormulaBroken": false }, "references": [ @@ -3577,7 +3995,7 @@ "dataType": "number", "filter": { "language": "kuery", - "query": "\"logstash.pipeline.plugin.filter.events.out\": *" + "query": "\"logstash.pipeline.plugin.filter.events.in\": *" }, "isBucketed": false, "label": "Part of Events Emitted per second", @@ -3586,7 +4004,7 @@ "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.filter.events.out" + "sourceField": "logstash.pipeline.plugin.filter.events.in" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { "customLabel": true, @@ -3685,13 +4103,13 @@ }, "gridData": { "h": 11, - "i": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", + "i": "d6db9db8-7da0-435c-8118-9d0c36d2573d", "w": 20, - "x": 28, - "y": 80 + "x": 8, + "y": 92 }, - "panelIndex": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", - "title": "Events Emitted per second", + "panelIndex": "d6db9db8-7da0-435c-8118-9d0c36d2573d", + "title": "Events Received per second", "type": "lens", "version": "8.10.1" }, @@ -3713,31 +4131,57 @@ "layers": { "03132fcd-13a1-4f64-88ad-31cbf8b4e043": { "columnOrder": [ - "ff88ade5-20cd-4210-b34e-7712c5da3acb", - "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f", + "d531162d-3ee2-4de1-8427-c57e8d679cde", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", "b02fb0ac-3ead-4546-ac07-2d21f54628dc", "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0", "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" ], "columns": { - "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f": { - "dataType": "date", + "2c04dbf8-9f1b-4d3b-a598-1503b9d97010": { + "dataType": "string", "isBucketed": true, - "label": "@timestamp", - "operationType": "date_histogram", + "label": "Top values of logstash.host.name + 2 others", + "operationType": "terms", "params": { - "dropPartials": false, - "includeEmptyRows": false, - "interval": "auto" + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of logstash.pipeline.total.events.out", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.plugin.filter.name", + "logstash.pipeline.plugin.filter.id" + ], + "size": 20 }, - "scale": "interval", - "sourceField": "@timestamp" + "scale": "ordinal", + "sourceField": "logstash.host.name" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dc": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Worker Utilization", + "label": "Events Emitted per second", "operationType": "formula", "params": { "format": { @@ -3746,31 +4190,36 @@ "decimals": 2 } }, - "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", + "formula": "counter_rate(last_value(logstash.pipeline.plugin.filter.events.out))", "isFormulaBroken": false }, "references": [ "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1" ], - "scale": "ratio" + "scale": "ratio", + "timeScale": "s" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0": { "customLabel": true, "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.plugin.filter.events.out\": *" + }, "isBucketed": false, - "label": "Part of Worker Millis per event", - "operationType": "max", + "label": "Part of Events Emitted per second", + "operationType": "last_value", "params": { - "emptyAsNull": false + "sortField": "@timestamp" }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + "sourceField": "logstash.pipeline.plugin.filter.events.out" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Worker Millis per event", + "label": "Part of Events Emitted per second", "operationType": "counter_rate", "references": [ "b02fb0ac-3ead-4546-ac07-2d21f54628dcX0" @@ -3778,48 +4227,18 @@ "scale": "ratio", "timeScale": "s" }, - "ff88ade5-20cd-4210-b34e-7712c5da3acb": { - "dataType": "string", + "d531162d-3ee2-4de1-8427-c57e8d679cde": { + "dataType": "date", "isBucketed": true, - "label": "Top values of logstash.host.name + 2 others", - "operationType": "terms", + "label": "@timestamp", + "operationType": "date_histogram", "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderAgg": { - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute: *" - }, - "isBucketed": false, - "label": "Last value of logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute", - "operationType": "last_value", - "params": { - "sortField": "@timestamp" - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute" - }, - "orderBy": { - "type": "custom" - }, - "orderDirection": "desc", - "otherBucket": false, - "parentFormat": { - "id": "multi_terms" - }, - "secondaryFields": [ - "logstash.pipeline.plugin.output.name", - "logstash.pipeline.plugin.output.id" - ], - "size": 10 + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" }, - "scale": "ordinal", - "sourceField": "logstash.host.name" + "scale": "interval", + "sourceField": "@timestamp" } }, "incompleteColumns": {}, @@ -3862,11 +4281,13 @@ ], "layerId": "03132fcd-13a1-4f64-88ad-31cbf8b4e043", "layerType": "data", - "position": "top", + "palette": { + "name": "default", + "type": "palette" + }, "seriesType": "line", - "showGridlines": false, - "splitAccessor": "ff88ade5-20cd-4210-b34e-7712c5da3acb", - "xAccessor": "5db84491-f1d2-48f0-8d6c-cb304e6b9e8f" + "splitAccessor": "2c04dbf8-9f1b-4d3b-a598-1503b9d97010", + "xAccessor": "d531162d-3ee2-4de1-8427-c57e8d679cde" } ], "legend": { @@ -3891,13 +4312,13 @@ }, "gridData": { "h": 11, - "i": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", + "i": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", "w": 20, - "x": 8, - "y": 91 + "x": 28, + "y": 92 }, - "panelIndex": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", - "title": "Worker Utilization", + "panelIndex": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6", + "title": "Events Emitted per second", "type": "lens", "version": "8.10.1" }, @@ -4083,8 +4504,8 @@ "h": 11, "i": "48f64446-f5ca-451d-8a59-bb12b4807381", "w": 20, - "x": 28, - "y": 91 + "x": 8, + "y": 103 }, "panelIndex": "48f64446-f5ca-451d-8a59-bb12b4807381", "title": "Worker Milliseconds per event", @@ -4122,7 +4543,7 @@ "i": "cfd22405-6008-4373-ad69-94e70427e259", "w": 24, "x": 17, - "y": 102 + "y": 114 }, "panelIndex": "cfd22405-6008-4373-ad69-94e70427e259", "title": "Plugins", @@ -4437,17 +4858,17 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Events output (per ms)", + "label": "Time spent (per event)", "operationType": "formula", "params": { "format": { "id": "number", "params": { "decimals": 2, - "suffix": "/ms" + "suffix": "ms" } }, - "formula": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.time.duration.ms)))", + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.time.duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.events.in)))", "isFormulaBroken": false }, "references": [ @@ -4458,24 +4879,20 @@ "e0728321-e2b4-44c6-b07b-a89f632f02e1X0": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.plugin.output.events.in\": *" - }, "isBucketed": false, - "label": "Part of Events output (per ms)", - "operationType": "last_value", + "label": "Part of Time spent (per event)", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.output.events.in" + "sourceField": "logstash.pipeline.plugin.output.time.duration.ms" }, "e0728321-e2b4-44c6-b07b-a89f632f02e1X1": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events output (per ms)", + "label": "Part of Time spent (per event)", "operationType": "counter_rate", "references": [ "e0728321-e2b4-44c6-b07b-a89f632f02e1X0" @@ -4487,7 +4904,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events output (per ms)", + "label": "Part of Time spent (per event)", "operationType": "cumulative_sum", "references": [ "e0728321-e2b4-44c6-b07b-a89f632f02e1X1" @@ -4497,24 +4914,20 @@ "e0728321-e2b4-44c6-b07b-a89f632f02e1X3": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "\"logstash.pipeline.plugin.output.time.duration.ms\": *" - }, "isBucketed": false, - "label": "Part of Events output (per ms)", - "operationType": "last_value", + "label": "Part of Time spent (per event)", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.output.time.duration.ms" + "sourceField": "logstash.pipeline.plugin.output.events.in" }, "e0728321-e2b4-44c6-b07b-a89f632f02e1X4": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events output (per ms)", + "label": "Part of Time spent (per event)", "operationType": "counter_rate", "references": [ "e0728321-e2b4-44c6-b07b-a89f632f02e1X3" @@ -4526,7 +4939,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events output (per ms)", + "label": "Part of Time spent (per event)", "operationType": "cumulative_sum", "references": [ "e0728321-e2b4-44c6-b07b-a89f632f02e1X4" @@ -4537,7 +4950,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Events output (per ms)", + "label": "Part of Time spent (per event)", "operationType": "math", "params": { "tinymathAst": { @@ -4546,11 +4959,11 @@ "e0728321-e2b4-44c6-b07b-a89f632f02e1X5" ], "location": { - "max": 174, + "max": 160, "min": 0 }, "name": "divide", - "text": "cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.events.in)))/cumulative_sum(counter_rate(last_value(logstash.pipeline.plugin.output.time.duration.ms)))", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.time.duration.ms)))/cumulative_sum(counter_rate(max(logstash.pipeline.plugin.output.events.in)))", "type": "function" } }, @@ -4761,11 +5174,11 @@ "params": { "colorStops": [ { - "color": "#cc5642", + "color": "#209280", "stop": 0 }, { - "color": "#e7664c", + "color": "#54b399", "stop": 20 }, { @@ -4773,11 +5186,11 @@ "stop": 40 }, { - "color": "#54b399", + "color": "#e7664c", "stop": 60 }, { - "color": "#209280", + "color": "#cc5642", "stop": 80 } ], @@ -4789,11 +5202,11 @@ "steps": 5, "stops": [ { - "color": "#cc5642", + "color": "#209280", "stop": 20 }, { - "color": "#e7664c", + "color": "#54b399", "stop": 40 }, { @@ -4801,11 +5214,11 @@ "stop": 60 }, { - "color": "#54b399", + "color": "#e7664c", "stop": 80 }, { - "color": "#209280", + "color": "#cc5642", "stop": 100 } ] @@ -4864,7 +5277,7 @@ "i": "5ea51e10-bede-40f6-9842-f840617bcc8b", "w": 40, "x": 8, - "y": 106 + "y": 118 }, "panelIndex": "5ea51e10-bede-40f6-9842-f840617bcc8b", "type": "lens", @@ -5149,7 +5562,7 @@ "i": "bb1876b7-7faa-4c37-a8af-c79a70149922", "w": 40, "x": 8, - "y": 122 + "y": 134 }, "panelIndex": "bb1876b7-7faa-4c37-a8af-c79a70149922", "title": "Average time spent processing event (ms)", @@ -5358,7 +5771,7 @@ "i": "c49e6ec7-e4a6-4938-b39b-d0c49b1ae114", "w": 20, "x": 8, - "y": 133 + "y": 145 }, "panelIndex": "c49e6ec7-e4a6-4938-b39b-d0c49b1ae114", "title": "Events received per second", @@ -5567,7 +5980,7 @@ "i": "231824a2-a183-439f-8e08-e1bad6c5e9db", "w": 20, "x": 28, - "y": 133 + "y": 145 }, "panelIndex": "231824a2-a183-439f-8e08-e1bad6c5e9db", "title": "Events emitted per second", @@ -5616,7 +6029,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Worker Millis per event", + "label": "Worker Utilization", "operationType": "formula", "params": { "format": { @@ -5625,7 +6038,7 @@ "decimals": 2 } }, - "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute))", + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", "isFormulaBroken": false }, "references": [ @@ -5643,7 +6056,7 @@ "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute" + "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { "customLabel": true, @@ -5770,13 +6183,13 @@ }, "gridData": { "h": 11, - "i": "09cd3d58-f4a5-4033-a011-15002b6649e7", + "i": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", "w": 20, "x": 8, - "y": 144 + "y": 156 }, - "panelIndex": "09cd3d58-f4a5-4033-a011-15002b6649e7", - "title": "Worker millseconds per event", + "panelIndex": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972", + "title": "Worker Utilization", "type": "lens", "version": "8.10.1" }, @@ -5822,7 +6235,7 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Worker Utilization", + "label": "Worker Millis per event", "operationType": "formula", "params": { "format": { @@ -5831,7 +6244,7 @@ "decimals": 2 } }, - "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute))", + "formula": "counter_rate(max(logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute))", "isFormulaBroken": false }, "references": [ @@ -5849,7 +6262,7 @@ "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute" + "sourceField": "logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute" }, "b02fb0ac-3ead-4546-ac07-2d21f54628dcX1": { "customLabel": true, @@ -5976,13 +6389,13 @@ }, "gridData": { "h": 11, - "i": "03d92828-8549-4cd9-a7ea-12f25502cef1", + "i": "09cd3d58-f4a5-4033-a011-15002b6649e7", "w": 20, "x": 28, - "y": 144 + "y": 156 }, - "panelIndex": "03d92828-8549-4cd9-a7ea-12f25502cef1", - "title": "Worker utilization", + "panelIndex": "09cd3d58-f4a5-4033-a011-15002b6649e7", + "title": "Worker millseconds per event", "type": "lens", "version": "8.10.1" }, @@ -6018,7 +6431,7 @@ "i": "31278c59-d923-4beb-800b-a8107e9064c5", "w": 24, "x": 16, - "y": 155 + "y": 167 }, "panelIndex": "31278c59-d923-4beb-800b-a8107e9064c5", "title": "Plugins", @@ -6605,7 +7018,7 @@ "i": "46fbd106-9dba-4c2c-b359-fd04c22e1314", "w": 40, "x": 8, - "y": 159 + "y": 171 }, "panelIndex": "46fbd106-9dba-4c2c-b359-fd04c22e1314", "type": "lens", @@ -6613,13 +7026,13 @@ } ], "timeRestore": false, - "title": "[Metrics Logstash] Logstash Plugins - Advanced View", + "title": "[Metrics Logstash] Logstash Single Pipeline View", "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:22:03.327Z", + "created_at": "2023-10-04T17:11:05.850Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", @@ -6641,6 +7054,11 @@ "name": "6da9466a-ede0-44b6-a2e7-d704065bb90b:indexpattern-datasource-layer-d6fe4fea-e92e-4042-9c92-0663b598a523", "type": "index-pattern" }, + { + "id": "logstash-sm-metrics", + "name": "7d85767a-5dbb-4704-8716-1d4c30a5f675:indexpattern-datasource-layer-4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "type": "index-pattern" + }, { "id": "logstash-sm-metrics", "name": "f03395de-a163-4204-ac82-83c0ea542eeb:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", @@ -6661,6 +7079,11 @@ "name": "5171ef08-d301-4a9c-832b-40f41c9ad24d:indexpattern-datasource-layer-a387e7a5-3021-4123-8be6-dc78275ee3cb", "type": "index-pattern" }, + { + "id": "logstash-sm-metrics", + "name": "03d92828-8549-4cd9-a7ea-12f25502cef1:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "type": "index-pattern" + }, { "id": "logstash-sm-metrics", "name": "9a34af7e-5f12-49b4-86e0-5577203711eb:indexpattern-datasource-layer-df79554f-08e3-4f7f-b438-80bc81e14637", @@ -6701,11 +7124,6 @@ "name": "55b1238e-1f8e-4b73-8dfc-906aa2eecbb6:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", "type": "index-pattern" }, - { - "id": "logstash-sm-metrics", - "name": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", - "type": "index-pattern" - }, { "id": "logstash-sm-metrics", "name": "48f64446-f5ca-451d-8a59-bb12b4807381:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", @@ -6733,12 +7151,12 @@ }, { "id": "logstash-sm-metrics", - "name": "09cd3d58-f4a5-4033-a011-15002b6649e7:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "name": "1c17de77-a7e5-4265-ada5-a7aaf0b6c972:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "03d92828-8549-4cd9-a7ea-12f25502cef1:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", + "name": "09cd3d58-f4a5-4033-a011-15002b6649e7:indexpattern-datasource-layer-03132fcd-13a1-4f64-88ad-31cbf8b4e043", "type": "index-pattern" }, { diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index 42d0ccbd1f1..7472139a4e9 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -24,6 +24,43 @@ "useMargins": true }, "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n**[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548)** \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\n\nOverview\n\nThis Dashboard gives an overall view of pipelines running on dashboards. \nClick on a pipeline in the list view to get more details on the running pipeline, and the plugins in that pipeline.", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 37, + "i": "3edb2e9f-2807-4e65-9adc-259c15debce9", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "3edb2e9f-2807-4e65-9adc-259c15debce9", + "title": "Logstash Nav Panel", + "type": "visualization", + "version": "8.10.1" + }, { "embeddableConfig": { "attributes": { @@ -437,10 +474,18 @@ "2d6ddf9d-fd47-4ec3-8e3c-fad74fbff7f1", "8b340f82-b329-4adc-8a42-82c2a8495d2c", "7f9470fb-8d50-485f-a388-9c18afe3e735", + "84007e47-0f47-4951-b26f-422e68b1fbac", "1c8d484c-64cf-4497-bede-36e69ad4047f", "5af7d5b3-3989-4473-89e7-17461f4b9c22", "d1d9a76c-a44f-4027-853e-263043f202a1", - "cc68d659-2c7d-4689-b30a-c536d451cf0f" + "cc68d659-2c7d-4689-b30a-c536d451cf0f", + "84007e47-0f47-4951-b26f-422e68b1fbacX0", + "84007e47-0f47-4951-b26f-422e68b1fbacX1", + "84007e47-0f47-4951-b26f-422e68b1fbacX2", + "84007e47-0f47-4951-b26f-422e68b1fbacX3", + "84007e47-0f47-4951-b26f-422e68b1fbacX4", + "84007e47-0f47-4951-b26f-422e68b1fbacX5", + "84007e47-0f47-4951-b26f-422e68b1fbacX6" ], "columns": { "1c8d484c-64cf-4497-bede-36e69ad4047f": { @@ -534,6 +579,125 @@ "scale": "ratio", "sourceField": "logstash.pipeline.total.events.out" }, + "84007e47-0f47-4951-b26f-422e68b1fbac": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time taken per event (ms)", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "isFormulaBroken": false + }, + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX6" + ], + "scale": "ratio" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.time.duration.ms" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "counter_rate", + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "cumulative_sum", + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX1" + ], + "scale": "ratio" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "counter_rate", + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "cumulative_sum", + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX4" + ], + "scale": "ratio" + }, + "84007e47-0f47-4951-b26f-422e68b1fbacX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time taken per event (ms)", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX2", + "84007e47-0f47-4951-b26f-422e68b1fbacX5" + ], + "location": { + "max": 147, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "type": "function" + } + }, + "references": [ + "84007e47-0f47-4951-b26f-422e68b1fbacX2", + "84007e47-0f47-4951-b26f-422e68b1fbacX5" + ], + "scale": "ratio" + }, "8b340f82-b329-4adc-8a42-82c2a8495d2c": { "customLabel": true, "dataType": "number", @@ -670,28 +834,33 @@ "columnId": "1c8d484c-64cf-4497-bede-36e69ad4047f", "isTransposed": false, "palette": { - "name": "positive", + "name": "status", "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, "stops": [ { - "color": "#d6e9e4", + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", "stop": 20 }, { - "color": "#aed3ca", + "color": "#d6bf57", "stop": 40 }, { - "color": "#85bdb1", + "color": "#e7664c", "stop": 60 }, { - "color": "#5aa898", + "color": "#cc5642", "stop": 80 - }, - { - "color": "#209280", - "stop": 100 } ] }, @@ -704,28 +873,72 @@ "columnId": "5af7d5b3-3989-4473-89e7-17461f4b9c22", "isTransposed": false, "palette": { - "name": "positive", + "name": "status", "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, "stops": [ { - "color": "#d6e9e4", + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", "stop": 20 }, { - "color": "#aed3ca", + "color": "#d6bf57", "stop": 40 }, { - "color": "#85bdb1", + "color": "#e7664c", "stop": 60 }, { - "color": "#5aa898", + "color": "#cc5642", "stop": 80 - }, + } + ] + }, + "type": "palette" + } + }, + { + "alignment": "center", + "colorMode": "cell", + "columnId": "84007e47-0f47-4951-b26f-422e68b1fbac", + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ { "color": "#209280", - "stop": 100 + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 } ] }, @@ -797,8 +1010,7 @@ "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", "66ce1290-0494-4de1-b877-e672f10deaf3", "735d813f-2024-473c-bc2c-69cbcfb5ff29", - "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", - "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" ], "columns": { "66ce1290-0494-4de1-b877-e672f10deaf3": { @@ -817,46 +1029,29 @@ "735d813f-2024-473c-bc2c-69cbcfb5ff29": { "customLabel": true, "dataType": "number", - "isBucketed": false, - "label": "Events Emitted Rate", - "operationType": "formula", - "params": { - "formula": "counter_rate(max(logstash.pipeline.total.events.out), kql='logstash.pipeline.total.events.in: *')", - "isFormulaBroken": false + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" }, + "isBucketed": false, + "label": "Events Received Rate", + "operationType": "counter_rate", "references": [ - "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" ], "scale": "ratio", "timeScale": "s" }, - "735d813f-2024-473c-bc2c-69cbcfb5ff29X0": { - "customLabel": true, + "88e7718d-261a-4cf0-8558-ae0bfbe0cae0": { "dataType": "number", "isBucketed": false, - "label": "Part of Events Emitted Rate", + "label": "Maximum of logstash.pipeline.total.events.in", "operationType": "max", "params": { - "emptyAsNull": false - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.total.events.out" - }, - "735d813f-2024-473c-bc2c-69cbcfb5ff29X1": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.pipeline.total.events.in: *" + "emptyAsNull": true }, - "isBucketed": false, - "label": "Part of Events Emitted Rate", - "operationType": "counter_rate", - "references": [ - "735d813f-2024-473c-bc2c-69cbcfb5ff29X0" - ], "scale": "ratio", - "timeScale": "s" + "sourceField": "logstash.pipeline.total.events.in" }, "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { "dataType": "string", @@ -933,14 +1128,14 @@ "hidePanelTitles": false }, "gridData": { - "h": 7, - "i": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", + "h": 8, + "i": "1a42a8c2-9823-4178-90ae-f3a403b63711", "w": 20, - "x": 8, - "y": 12 + "x": 28, + "y": 21 }, - "panelIndex": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", - "title": "Events Emitted Rate ", + "panelIndex": "1a42a8c2-9823-4178-90ae-f3a403b63711", + "title": "Events Received Rate", "type": "lens", "version": "8.10.1" }, @@ -965,7 +1160,8 @@ "c10baf86-fe98-4d07-b5da-cc4c8b0f901e", "66ce1290-0494-4de1-b877-e672f10deaf3", "735d813f-2024-473c-bc2c-69cbcfb5ff29", - "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0", + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" ], "columns": { "66ce1290-0494-4de1-b877-e672f10deaf3": { @@ -984,29 +1180,46 @@ "735d813f-2024-473c-bc2c-69cbcfb5ff29": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.pipeline.total.events.in: *" - }, "isBucketed": false, - "label": "Events Received Rate", - "operationType": "counter_rate", + "label": "Events Emitted Rate", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.total.events.out), kql='logstash.pipeline.total.events.in: *')", + "isFormulaBroken": false + }, "references": [ - "88e7718d-261a-4cf0-8558-ae0bfbe0cae0" + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1" ], "scale": "ratio", "timeScale": "s" }, - "88e7718d-261a-4cf0-8558-ae0bfbe0cae0": { + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0": { + "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Maximum of logstash.pipeline.total.events.in", + "label": "Part of Events Emitted Rate", "operationType": "max", "params": { - "emptyAsNull": true + "emptyAsNull": false }, "scale": "ratio", - "sourceField": "logstash.pipeline.total.events.in" + "sourceField": "logstash.pipeline.total.events.out" + }, + "735d813f-2024-473c-bc2c-69cbcfb5ff29X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.pipeline.total.events.in: *" + }, + "isBucketed": false, + "label": "Part of Events Emitted Rate", + "operationType": "counter_rate", + "references": [ + "735d813f-2024-473c-bc2c-69cbcfb5ff29X0" + ], + "scale": "ratio", + "timeScale": "s" }, "c10baf86-fe98-4d07-b5da-cc4c8b0f901e": { "dataType": "string", @@ -1083,14 +1296,14 @@ "hidePanelTitles": false }, "gridData": { - "h": 7, - "i": "1a42a8c2-9823-4178-90ae-f3a403b63711", + "h": 8, + "i": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", "w": 20, - "x": 28, - "y": 12 + "x": 8, + "y": 21 }, - "panelIndex": "1a42a8c2-9823-4178-90ae-f3a403b63711", - "title": "Events Received Rate", + "panelIndex": "09177cfd-5361-4e91-b7c3-64eeb7837ce8", + "title": "Events Emitted Rate ", "type": "lens", "version": "8.10.1" }, @@ -1232,11 +1445,11 @@ "hidePanelTitles": false }, "gridData": { - "h": 7, + "h": 8, "i": "4017b8f5-f8f2-46b9-86fe-501f46d551c3", "w": 20, "x": 8, - "y": 19 + "y": 29 }, "panelIndex": "4017b8f5-f8f2-46b9-86fe-501f46d551c3", "title": "Persistent Queue Event Count", @@ -1483,11 +1696,11 @@ "hidePanelTitles": false }, "gridData": { - "h": 7, + "h": 8, "i": "d290b2f1-1769-41f4-9d29-41755eef8871", "w": 20, "x": 28, - "y": 19 + "y": 29 }, "panelIndex": "d290b2f1-1769-41f4-9d29-41755eef8871", "title": "Persistent Queue Disk Utilization", @@ -1496,39 +1709,250 @@ }, { "embeddableConfig": { - "enhancements": {}, - "savedVis": { - "data": { - "aggs": [], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" + "attributes": { + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "4d13c1a7-6b92-47fe-8baf-7ca505bd9fda": { + "columnOrder": [ + "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3", + "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6" + ], + "columns": { + "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline Name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.name" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Average time processing per event", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": "ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "isFormulaBroken": false + }, + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.time.duration.ms" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "counter_rate", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "cumulative_sum", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX1" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "counter_rate", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "cumulative_sum", + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX4" + ], + "scale": "ratio" + }, + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average time processing per event", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5" + ], + "location": { + "max": 147, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(logstash.pipeline.total.time.duration.ms)))/cumulative_sum(counter_rate((max(logstash.pipeline.total.events.out))))", + "type": "function" + } + }, + "references": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX2", + "d9cd1ee6-586c-41b4-9d5f-976cd8eececfX5" + ], + "scale": "ratio" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "d9cd1ee6-586c-41b4-9d5f-976cd8eececf" + ], + "layerId": "4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "31515c2b-a1f2-4da6-a5f7-f6f2ad8bc7d3", + "xAccessor": "1c9f4a0d-3d6f-4126-afbb-ba4745e565d5" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" } }, - "description": "", - "params": { - "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n**[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548)** \n[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)\n\n\nOverview\n\nThis Dashboard gives an overall view of pipelines running on dashboards. \nClick on a pipeline in the list view to get more details on the running pipeline, and the plugins in that pipeline.", - "openLinksInNewTab": false - }, - "title": "", - "type": "markdown", - "uiState": {} - } + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false }, "gridData": { - "h": 26, - "i": "3edb2e9f-2807-4e65-9adc-259c15debce9", - "w": 8, - "x": 0, - "y": 0 + "h": 9, + "i": "6609ea37-a262-4072-ac19-90aabff49e12", + "w": 40, + "x": 8, + "y": 12 }, - "panelIndex": "3edb2e9f-2807-4e65-9adc-259c15debce9", - "title": "Logstash Nav Panel", - "type": "visualization", + "panelIndex": "6609ea37-a262-4072-ac19-90aabff49e12", + "title": "Average Time processed per event (ms)", + "type": "lens", "version": "8.10.1" } ], @@ -1537,9 +1961,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:22:03.327Z", + "created_at": "2023-10-04T17:11:25.489Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", @@ -1573,12 +1997,12 @@ }, { "id": "logstash-sm-metrics", - "name": "09177cfd-5361-4e91-b7c3-64eeb7837ce8:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "name": "1a42a8c2-9823-4178-90ae-f3a403b63711:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", "type": "index-pattern" }, { "id": "logstash-sm-metrics", - "name": "1a42a8c2-9823-4178-90ae-f3a403b63711:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", + "name": "09177cfd-5361-4e91-b7c3-64eeb7837ce8:indexpattern-datasource-layer-0dc38ed3-eda8-48d3-85d1-454d187be2f1", "type": "index-pattern" }, { @@ -1596,6 +2020,11 @@ "name": "d290b2f1-1769-41f4-9d29-41755eef8871:43c6d3b9-ff0a-40c8-bc5b-07585d2acf47", "type": "index-pattern" }, + { + "id": "logstash-sm-metrics", + "name": "6609ea37-a262-4072-ac19-90aabff49e12:indexpattern-datasource-layer-4d13c1a7-6b92-47fe-8baf-7ca505bd9fda", + "type": "index-pattern" + }, { "id": "logstash-sm-metrics", "name": "controlGroup_94cc8a2a-a81e-451b-891b-407075069331:optionsListDataView", diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index e14b046db0f..5bea566b7ad 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -909,7 +909,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-03T21:22:03.327Z", + "created_at": "2023-10-04T16:10:54.914Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", "managed": true, "references": [ From 48bd0e82269bd8d8ebed808ee6d0bcac16c78928 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 4 Oct 2023 20:14:56 -0400 Subject: [PATCH 09/25] Bump required kibana to 8.10.1 --- packages/logstash/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/logstash/manifest.yml b/packages/logstash/manifest.yml index 02b197095d8..a494bd8aa5d 100644 --- a/packages/logstash/manifest.yml +++ b/packages/logstash/manifest.yml @@ -12,7 +12,7 @@ format_version: 1.0.0 license: basic categories: ["elastic_stack"] conditions: - kibana.version: ^8.7.1 + kibana.version: ^8.10.1 owner: github: elastic/infra-monitoring-ui screenshots: From f2c8d43f1c7070f53ece534a754cc3c19c6d4755 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 09:37:57 -0400 Subject: [PATCH 10/25] Rename stream.yml.hbs to cel.yml.hbs --- .../node_cel/agent/stream/{stream.yml.hbs => cel.yml.hbs} | 0 packages/logstash/data_stream/node_cel/manifest.yml | 2 +- .../pipeline/agent/stream/{stream.yml.hbs => cel.yml.hbs} | 0 packages/logstash/data_stream/pipeline/manifest.yml | 2 +- .../plugins/agent/stream/{stream.yml.hbs => cel.yml.hbs} | 0 packages/logstash/data_stream/plugins/manifest.yml | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) rename packages/logstash/data_stream/node_cel/agent/stream/{stream.yml.hbs => cel.yml.hbs} (100%) rename packages/logstash/data_stream/pipeline/agent/stream/{stream.yml.hbs => cel.yml.hbs} (100%) rename packages/logstash/data_stream/plugins/agent/stream/{stream.yml.hbs => cel.yml.hbs} (100%) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs similarity index 100% rename from packages/logstash/data_stream/node_cel/agent/stream/stream.yml.hbs rename to packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs diff --git a/packages/logstash/data_stream/node_cel/manifest.yml b/packages/logstash/data_stream/node_cel/manifest.yml index e868d1097bf..0308ce334ed 100644 --- a/packages/logstash/data_stream/node_cel/manifest.yml +++ b/packages/logstash/data_stream/node_cel/manifest.yml @@ -11,7 +11,7 @@ streams: enabled: false title: "Collect Node Metrics" description: "Collect Metrics related to Nodes running Logstash" - template_path: stream.yml.hbs + template_path: cel.yml.hbs vars: - name: period type: text diff --git a/packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs similarity index 100% rename from packages/logstash/data_stream/pipeline/agent/stream/stream.yml.hbs rename to packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs diff --git a/packages/logstash/data_stream/pipeline/manifest.yml b/packages/logstash/data_stream/pipeline/manifest.yml index 2aef1281593..ddbb9b41cd1 100644 --- a/packages/logstash/data_stream/pipeline/manifest.yml +++ b/packages/logstash/data_stream/pipeline/manifest.yml @@ -11,7 +11,7 @@ streams: enabled: false title: "Collect Pipeline Metrics" description: "Collect Metrics related to Logstash Pipeline usage" - template_path: stream.yml.hbs + template_path: cel.yml.hbs vars: - name: period type: text diff --git a/packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs similarity index 100% rename from packages/logstash/data_stream/plugins/agent/stream/stream.yml.hbs rename to packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs diff --git a/packages/logstash/data_stream/plugins/manifest.yml b/packages/logstash/data_stream/plugins/manifest.yml index dc96ee85294..7fa3aca5b64 100644 --- a/packages/logstash/data_stream/plugins/manifest.yml +++ b/packages/logstash/data_stream/plugins/manifest.yml @@ -11,7 +11,7 @@ streams: enabled: false title: "Collect Plugin Metrics" description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate" - template_path: stream.yml.hbs + template_path: cel.yml.hbs vars: - name: period type: text From 0d6b93935d2418e549e1ceb9a910c0809a8972bb Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 09:46:00 -0400 Subject: [PATCH 11/25] Simlify mapping into events field in CEL scripts --- .../logstash/data_stream/node_cel/agent/stream/cel.yml.hbs | 2 +- .../logstash/data_stream/pipeline/agent/stream/cel.yml.hbs | 4 ++-- .../logstash/data_stream/plugins/agent/stream/cel.yml.hbs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs index dfe447a8749..c03f6ec7fda 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs @@ -64,7 +64,7 @@ program: | }}) ) .as(eve, { - "events":eve.map(each, eve), + "events":[eve], "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null }, diff --git a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs index 109dec17a19..22d814b2bb7 100644 --- a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs @@ -77,8 +77,8 @@ program: | }) ) )) - .as(eve, { + .as(pipelines, { "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null }, - "events":eve.map(each, {"logstash":{"pipeline":each}})}) \ No newline at end of file + "events":pipelines.map(pipeline, {"logstash":{"pipeline":pipeline}})}) \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index d2ba5271de0..b7e6dd539c1 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -167,8 +167,8 @@ program: | } }.drop_empty() ) - }).collate(["filters", "outputs", "inputs", "codecs"])).as(eve, { + }).collate(["filters", "outputs", "inputs", "codecs"])).as(plugins, { "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null }, - "events":eve.map(each, {"logstash":{"pipeline":each}})}) \ No newline at end of file + "events":plugins.map(plugin, {"logstash":{"pipeline":plugin}})}) \ No newline at end of file From 596f0d853817d9bee5343833e88db15dba0856f9 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 11:53:09 -0400 Subject: [PATCH 12/25] Tidy code by using dot notation --- .../node_cel/agent/stream/cel.yml.hbs | 36 +++++++++---------- .../plugins/agent/stream/cel.yml.hbs | 30 ++++++++-------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs index c03f6ec7fda..4dac66716f1 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs @@ -35,30 +35,30 @@ program: | .decode_json() .as(body, {"logstash":{"node":{"stats":{ - "events":body['events'], + "events":body.events, "jvm":{ - "uptime_in_millis":body['jvm']['uptime_in_millis'], + "uptime_in_millis":body.jvm.uptime_in_millis, "mem":body['jvm']['mem'].drop("pools"), - "threads":body['jvm']['threads'] + "threads":body.jvm.threads }, - "queue":body['queue'], - "reloads":body['reloads'], - "process":body['process'], + "queue":body.queue, + "reloads":body.reloads, + "process":body.process, "os":{ - "cpu":body['process']['cpu'], - "cgroup":has(body.os.group) ? body['os']['cgroup'] : {}, + "cpu":body.process.cpu, + "cgroup":has(body.os.group) ? body.os.cgroup : {}, }, "logstash":{ - "ephemeral_id":body['ephemeral_id'], - "host":body['host'], - "http_address":body['http_address'], - "name":body['name'], - "pipeline":body['pipeline'], - "pipelines":body['pipelines'].map(pipeline, [pipeline]).flatten(), - "snapshot":body['snapshot'], - "status":body['status'], - "uuid":body['id'], - "version":body['version'], + "ephemeral_id":body.ephemeral_id, + "host":body.host, + "http_address":body.http_address, + "name":body.name, + "pipeline":body.pipeline, + "pipelines":body.pipelines.map(pipeline, [pipeline]).flatten(), + "snapshot":body.snapshot, + "status":body.status, + "uuid":body.id, + "version":body.version, } }} }}) diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index b7e6dd539c1..dfa79209455 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -66,16 +66,16 @@ program: | "name":event.name, "id":event.hash, "host":event.host, - "elasticsearch.cluster.id":event['es_cluster_id'], + "elasticsearch.cluster.id":event.es_cluster_id, "plugin":{ "type":"input", "input":{ "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == input.id), tuple.cluster_id), "name":input.name, "id":input.id, - "flow": has(input.flow) ? input["flow"] : {}, + "flow": has(input.flow) ? input.flow : {}, "events":{ - "out":input.events["out"], + "out":input.events.out, }, "time":{ "queue_push_duration":{ @@ -91,22 +91,22 @@ program: | "name":event.name, "id":event.hash, "host":event.host, - "elasticsearch.cluster.id":event['es_cluster_id'], + "elasticsearch.cluster.id":event.es_cluster_id, "plugin":{ "type":"codec", "codec":{ "id":codec.id, "name":codec.name, - "flow": has(codec.flow) ? codec["flow"] : {}, + "flow": has(codec.flow) ? codec.flow : {}, "decode":{ "duration":{ "ms":codec.decode.duration_in_millis }, - "in":codec.decode["writes_in"], - "out":codec.decode["out"], + "in":codec.decode.writes_in, + "out":codec.decode.out, }, "encode":{ - "in":codec.encode["writes_in"], + "in":codec.encode.writes_in, "duration":{ "ms":codec.encode.duration_in_millis } @@ -120,17 +120,17 @@ program: | "name":event.name, "id":event.hash, "host":event.host, - "elasticsearch.cluster.id":event['es_cluster_id'], + "elasticsearch.cluster.id":event.es_cluster_id, "plugin":{ "type":"filter", "filter":{ "id":filter.id, "name":filter.name, "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == filter.id), tuple.cluster_id), - "flow": has(filter.flow) ? filter["flow"] : {}, + "flow": has(filter.flow) ? filter.flow : {}, "events":{ - "in":filter.events["in"], - "out":filter.events["out"], + "in":filter.events['in'], + "out":filter.events.out, }, "time":{ "duration":{ @@ -153,10 +153,10 @@ program: | "id":output.id, "name":output.name, "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == output.id), tuple.cluster_id), - "flow": has(output.flow) ? output["flow"] : {}, + "flow": has(output.flow) ? output.flow : {}, "events":{ - "in":output.events["in"], - "out":output.events["out"], + "in":output.events['in'], + "out":output.events.out, }, "time":{ "duration":{ From 7909f7ec6342835d58042ebf40ab1edbb680ae86 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 11:54:47 -0400 Subject: [PATCH 13/25] Tidy up Dashboard --- ...-79270240-48ee-11ee-8cb5-99927777c522.json | 2 +- ...-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 2 +- ...-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 2 +- ...-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 6 +- ...-c0594170-526a-11ee-9ecc-31444cb79548.json | 65 +++++++++++++++---- ...-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 2 +- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index f9a73814847..fc966b0d1a3 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1156,7 +1156,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T16:10:54.914Z", + "created_at": "2023-10-05T15:38:49.040Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index 79c148f4683..08668a06609 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -1570,7 +1570,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T16:10:54.914Z", + "created_at": "2023-10-05T15:38:49.040Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index eca5ef5040d..bd14c21ada4 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -1897,7 +1897,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T16:10:54.914Z", + "created_at": "2023-10-05T15:38:49.040Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index 974ab0a270f..3e363461a81 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -242,7 +242,7 @@ "query": "\"logstash.pipeline.total.events.in\": *" }, "isBucketed": false, - "label": "Events received by pipeline", + "label": "Total events received by pipeline", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -327,7 +327,7 @@ "query": "\"logstash.pipeline.total.events.out\": *" }, "isBucketed": false, - "label": "Events emitted from pipeline", + "label": "Total events emitted from pipeline", "operationType": "last_value", "params": { "sortField": "@timestamp" @@ -7030,7 +7030,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T17:11:05.850Z", + "created_at": "2023-10-05T15:52:49.964Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", "managed": false, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index 7472139a4e9..3f055fbfd21 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -485,7 +485,10 @@ "84007e47-0f47-4951-b26f-422e68b1fbacX3", "84007e47-0f47-4951-b26f-422e68b1fbacX4", "84007e47-0f47-4951-b26f-422e68b1fbacX5", - "84007e47-0f47-4951-b26f-422e68b1fbacX6" + "84007e47-0f47-4951-b26f-422e68b1fbacX6", + "8b340f82-b329-4adc-8a42-82c2a8495d2cX0", + "8b340f82-b329-4adc-8a42-82c2a8495d2cX1", + "d407e28c-5456-45a7-8dc3-fdb1407c9d59" ], "columns": { "1c8d484c-64cf-4497-bede-36e69ad4047f": { @@ -521,10 +524,10 @@ "includeIsRegex": false, "missingBucket": false, "orderBy": { - "columnId": "7f9470fb-8d50-485f-a388-9c18afe3e735", - "type": "column" + "fallback": true, + "type": "alphabetical" }, - "orderDirection": "desc", + "orderDirection": "asc", "otherBucket": true, "parentFormat": { "id": "terms" @@ -572,12 +575,11 @@ "dataType": "number", "isBucketed": false, "label": "Events emitted", - "operationType": "max", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "logstash.pipeline.total.events.out" + "operationType": "counter_rate", + "references": [ + "d407e28c-5456-45a7-8dc3-fdb1407c9d59" + ], + "scale": "ratio" }, "84007e47-0f47-4951-b26f-422e68b1fbac": { "customLabel": true, @@ -703,13 +705,40 @@ "dataType": "number", "isBucketed": false, "label": "Events received", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(logstash.pipeline.total.events.in))", + "isFormulaBroken": false + }, + "references": [ + "8b340f82-b329-4adc-8a42-82c2a8495d2cX1" + ], + "scale": "ratio" + }, + "8b340f82-b329-4adc-8a42-82c2a8495d2cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Received", "operationType": "max", "params": { - "emptyAsNull": true + "emptyAsNull": false }, "scale": "ratio", "sourceField": "logstash.pipeline.total.events.in" }, + "8b340f82-b329-4adc-8a42-82c2a8495d2cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events Received", + "operationType": "counter_rate", + "references": [ + "8b340f82-b329-4adc-8a42-82c2a8495d2cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, "cc68d659-2c7d-4689-b30a-c536d451cf0f": { "dataType": "number", "isBucketed": false, @@ -731,6 +760,18 @@ }, "scale": "ratio", "sourceField": "logstash.pipeline.total.time.duration.ms" + }, + "d407e28c-5456-45a7-8dc3-fdb1407c9d59": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Events emitted", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.total.events.out" } }, "incompleteColumns": {}, @@ -1961,7 +2002,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T17:11:25.489Z", + "created_at": "2023-10-05T15:55:00.707Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", "managed": false, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 5bea566b7ad..43970a5ae2d 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -909,7 +909,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-04T16:10:54.914Z", + "created_at": "2023-10-05T15:38:49.040Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", "managed": true, "references": [ From 8cb394b04ccee1db761e377a5ed806cb9ce378b6 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 12:09:39 -0400 Subject: [PATCH 14/25] Clean up 'with' functions --- .../plugins/agent/stream/cel.yml.hbs | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index dfa79209455..2059f8b3213 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -35,24 +35,12 @@ program: | .decode_json().as(body, body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name}.with(body.pipelines[pipeline_name]) .with({ - "es_cluster_id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))) - }) - .with({ - "es_cluster_id_map":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), {"plugin_id":each.id, "cluster_id":each.cluster_uuid}))) - }) - .with({ - "outputs":body.pipelines[pipeline_name]["plugins"]["outputs"] - }) - .with({ - "inputs":body.pipelines[pipeline_name]["plugins"]["inputs"] - }) - .with({ - "filters":body.pipelines[pipeline_name]["plugins"]["filters"] - }) - .with({ - "codecs":body.pipelines[pipeline_name]["plugins"]["codecs"] - }) - .with({ + "es_cluster_id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))), + "es_cluster_id_map":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), {"plugin_id":each.id, "cluster_id":each.cluster_uuid}))), + "outputs":body.pipelines[pipeline_name]["plugins"]["outputs"], + "inputs":body.pipelines[pipeline_name]["plugins"]["inputs"], + "filters":body.pipelines[pipeline_name]["plugins"]["filters"], + "codecs":body.pipelines[pipeline_name]["plugins"]["codecs"], "host":{ "name":body.name, "address":body.http_address, From 3e7112948a37e247bb0c07bbc84c516aba3e7b96 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 15:14:22 -0400 Subject: [PATCH 15/25] Set fields that already exist in ECS to `external: ECS` --- .../data_stream/node_cel/fields/agent.yml | 117 ++++-------------- .../node_cel/fields/base-fields.yml | 9 +- .../data_stream/node_cel/fields/ecs.yml | 3 +- packages/logstash/docs/README.md | 22 ++-- 4 files changed, 40 insertions(+), 111 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/fields/agent.yml b/packages/logstash/data_stream/node_cel/fields/agent.yml index d0032fb9fff..19f18761f64 100644 --- a/packages/logstash/data_stream/node_cel/fields/agent.yml +++ b/packages/logstash/data_stream/node_cel/fields/agent.yml @@ -1,141 +1,74 @@ - name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - name: account.id - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + external: ecs - name: availability_zone - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. + external: ecs - name: instance.id - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. + external: ecs - name: instance.name - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. + external: ecs - name: machine.type - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. + external: ecs - name: provider - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + external: ecs - name: region - type: keyword - ignore_above: 1024 - description: Region in which this host is running. + external: ecs - name: project.id - type: keyword - description: Name of the project in Google Cloud. + external: ecs - name: image.id type: keyword description: Image ID for the cloud instance. - name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." type: group fields: - name: id - type: keyword - ignore_above: 1024 - description: Unique container id. + external: ecs - name: image.name - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. + external: ecs - name: labels - type: object - object_type: keyword - description: Image labels. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: Container name. + external: ecs - name: host - title: Host - group: 2 - description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - name: architecture - type: keyword - ignore_above: 1024 - description: Operating system architecture. + external: ecs - name: domain - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - default_field: false + external: ecs - name: hostname - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + external: ecs - name: id - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + external: ecs - name: ip - type: ip - description: Host ip addresses. + external: ecs - name: mac - type: keyword - ignore_above: 1024 - description: Host mac addresses. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + external: ecs - name: os.family - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). + external: ecs - name: os.kernel - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. + external: ecs - name: os.name - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. + external: ecs - name: os.platform - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). + external: ecs - name: os.version - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. + external: ecs - name: type - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + external: ecs - name: containerized type: boolean description: > If the host is a container. - - name: os.build type: keyword description: > OS build information. - - name: os.codename type: keyword description: > OS codename, if any. - diff --git a/packages/logstash/data_stream/node_cel/fields/base-fields.yml b/packages/logstash/data_stream/node_cel/fields/base-fields.yml index 4227333a0d2..1bfb71226cf 100644 --- a/packages/logstash/data_stream/node_cel/fields/base-fields.yml +++ b/packages/logstash/data_stream/node_cel/fields/base-fields.yml @@ -1,12 +1,9 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace - type: constant_keyword - description: Data stream namespace. + external: ecs - name: service.hostname type: keyword description: Hostname of the service diff --git a/packages/logstash/data_stream/node_cel/fields/ecs.yml b/packages/logstash/data_stream/node_cel/fields/ecs.yml index aad92fac0ef..d9087c71721 100644 --- a/packages/logstash/data_stream/node_cel/fields/ecs.yml +++ b/packages/logstash/data_stream/node_cel/fields/ecs.yml @@ -7,8 +7,7 @@ - name: service.version external: ecs - name: service.address - type: keyword - description: Address where data about this service was collected from. + external: ecs - name: service.name external: ecs - name: process.pid diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 32593b3cba8..9acc3e101b1 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -561,22 +561,22 @@ This is the `node` dataset, which drives the Node dashboard pages. |---|---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | cloud.instance.id | Instance ID of the host machine. | keyword | | | cloud.instance.name | Instance name of the host machine. | keyword | | | cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | Name of the project in Google Cloud. | keyword | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | -| cloud.region | Region in which this host is running. | keyword | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | | cluster_uuid | | alias | | | container.id | Unique container id. | keyword | | | container.image.name | Name of the image the container was built on. | keyword | | | container.labels | Image labels. | object | | | container.name | Container name. | keyword | | -| data_stream.dataset | Data stream dataset. | constant_keyword | | -| data_stream.namespace | Data stream namespace. | constant_keyword | | -| data_stream.type | Data stream type. | constant_keyword | | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | error.message | Error message. | match_only_text | | | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | @@ -588,14 +588,14 @@ This is the `node` dataset, which drives the Node dashboard pages. | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | host.ip | Host ip addresses. | ip | | -| host.mac | Host mac addresses. | keyword | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | host.os.kernel | Operating system kernel version as a raw string. | keyword | | | host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | host.os.version | Operating system version as a raw string. | keyword | | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | @@ -634,11 +634,11 @@ This is the `node` dataset, which drives the Node dashboard pages. | logstash.node.stats.logstash.uuid | | keyword | | | logstash.node.stats.logstash.version | | keyword | | | logstash.node.stats.os.cgroup.cpu.cfs_quota_micros | | long | gauge | -| logstash.node.stats.os.cgroup.cpu.control_group | | keyword | | +| logstash.node.stats.os.cgroup.cpu.control_group | | text | | | logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | gauge | | logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | counter | | logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | counter | -| logstash.node.stats.os.cgroup.cpuacct.control_group | | keyword | | +| logstash.node.stats.os.cgroup.cpuacct.control_group | | text | | | logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | counter | | logstash.node.stats.os.cpu.load_average.15m | | half_float | gauge | | logstash.node.stats.os.cpu.load_average.1m | | half_float | gauge | @@ -674,7 +674,7 @@ This is the `node` dataset, which drives the Node dashboard pages. | logstash.node.stats.timestamp | | date | | | logstash.pipeline.name | | alias | | | process.pid | Process id. | long | | -| service.address | Address where data about this service was collected from. | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.hostname | Hostname of the service | keyword | | | service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | From ad027b3611eef81b311be3da43e77ebbc00ec1e9 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 15:43:31 -0400 Subject: [PATCH 16/25] Make control group field types consistent --- packages/logstash/data_stream/node_cel/fields/fields.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/fields/fields.yml b/packages/logstash/data_stream/node_cel/fields/fields.yml index 408931027f9..c0061151c98 100644 --- a/packages/logstash/data_stream/node_cel/fields/fields.yml +++ b/packages/logstash/data_stream/node_cel/fields/fields.yml @@ -168,7 +168,7 @@ type: group fields: - name: control_group - type: keyword + type: text - name: usage_nanos type: long metric_type: counter @@ -176,7 +176,7 @@ type: group fields: - name: control_group - type: keyword + type: text - name: cfs_quota_micros type: long metric_type: gauge From a4ba6b4e5bd7a012bbbf9943c4ee845e91a0b1bd Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 16:05:51 -0400 Subject: [PATCH 17/25] Remove unnecessary drop_empty() commands --- .../logstash/data_stream/plugins/agent/stream/cel.yml.hbs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index 2059f8b3213..ef700487f0f 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -70,7 +70,7 @@ program: | "ms":input.events.queue_push_duration_in_millis } } - }.drop_empty() + } } }.drop_empty() ), @@ -125,7 +125,7 @@ program: | "ms":filter.events.duration_in_millis } } - }.drop_empty() + } } }.drop_empty() ), @@ -151,7 +151,7 @@ program: | "ms":output.events.duration_in_millis } } - }.drop_empty() + } } }.drop_empty() ) From 400ab49c1f10372351c78f13ef2b3029fabd45ec Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 17:37:57 -0400 Subject: [PATCH 18/25] Simplify auth Using the `get()` method instead of `get_request(url).do_request()` enables the use of basic auth by simply specifying `auth.basic` in the yaml definition allowing a whole bunch of code to be removed from the script, and only needing to redact the passwords, rather than use a blanket redaction. --- .../node_cel/agent/stream/cel.yml.hbs | 40 ++++++------------- .../pipeline/agent/stream/cel.yml.hbs | 32 +++++---------- .../plugins/agent/stream/cel.yml.hbs | 32 +++++---------- 3 files changed, 32 insertions(+), 72 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs index 4dac66716f1..cf3e28f7366 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs @@ -6,34 +6,21 @@ resource.ssl: {{resource_ssl}} {{/if}} -state: - username: {{username}} - password: {{password}} +{{#if username}} +auth.basic.user: {{escape_string username}} +{{/if}} +{{#if password}} +auth.basic.password: {{escape_string password}} +{{/if}} redact: - fields: ~ + fields: + - password program: | - (has(state.username) && state.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], - } - }) - : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], - } - } - ) - : - get_request(state.url) - ) - ) - .do_request().as(resp, bytes(resp.Body) - .decode_json() - .as(body, + get(state.url) + .as(resp, bytes(resp.Body) + .decode_json().as(body, {"logstash":{"node":{"stats":{ "events":body.events, "jvm":{ @@ -64,8 +51,5 @@ program: | }}) ) .as(eve, { - "events":[eve], - "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, - "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null - }, + "events":[eve] }) \ No newline at end of file diff --git a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs index 22d814b2bb7..ef752867417 100644 --- a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs @@ -6,32 +6,20 @@ resource.ssl: {{resource_ssl}} {{/if}} -state: - username: {{username}} - password: {{password}} +{{#if username}} +auth.basic.user: {{escape_string username}} +{{/if}} +{{#if password}} +auth.basic.password: {{escape_string password}} +{{/if}} redact: - fields: ~ + fields: + - password program: | - (has(state.username) && state.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], - } - }) - : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], - } - } - ) - : - get_request(state.url) - ) - ) - .do_request().as(resp, bytes(resp.Body) + get(state.url) + .as(resp, bytes(resp.Body) .decode_json().as(body, body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name} .with({ diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index ef700487f0f..96cba6d659e 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -6,32 +6,20 @@ resource.ssl: {{resource_ssl}} {{/if}} -state: - username: {{username}} - password: {{password}} +{{#if username}} +auth.basic.user: {{escape_string username}} +{{/if}} +{{#if password}} +auth.basic.password: {{escape_string password}} +{{/if}} redact: - fields: ~ + fields: + - password program: | - (has(state.username) && state.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.username+":"+state.password))], - } - }) - : (has(state.cursor) && has(state.cursor.username) && state.cursor.username != null ? - get_request(state.url).with({ - "Header":{ - "Authorization": ["Basic "+string(base64(state.cursor.username+":"+state.cursor.password))], - } - } - ) - : - get_request(state.url) - ) - ) - .do_request().as(resp, bytes(resp.Body) + get(state.url) + .as(resp, bytes(resp.Body) .decode_json().as(body, body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name}.with(body.pipelines[pipeline_name]) .with({ From 4fa4eac7e0cc7dd34315b7694868ba29bda3f3e8 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 5 Oct 2023 19:06:36 -0400 Subject: [PATCH 19/25] Futher tidying after code review comments --- .../node_cel/agent/stream/cel.yml.hbs | 5 +-- .../pipeline/agent/stream/cel.yml.hbs | 37 ++++++++----------- .../plugins/agent/stream/cel.yml.hbs | 20 ++++------ 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs index cf3e28f7366..f289951d08e 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs @@ -14,8 +14,7 @@ auth.basic.password: {{escape_string password}} {{/if}} redact: - fields: - - password + fields: ~ program: | get(state.url) @@ -25,7 +24,7 @@ program: | "events":body.events, "jvm":{ "uptime_in_millis":body.jvm.uptime_in_millis, - "mem":body['jvm']['mem'].drop("pools"), + "mem":body.jvm['mem'].drop("pools"), "threads":body.jvm.threads }, "queue":body.queue, diff --git a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs index ef752867417..92010ea77e1 100644 --- a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs @@ -14,8 +14,8 @@ auth.basic.password: {{escape_string password}} {{/if}} redact: - fields: - - password + fields: ~ + program: | get(state.url) @@ -23,42 +23,38 @@ program: | .decode_json().as(body, body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name} .with({ - "elasticsearch.cluster.id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))) - }) - .with({ + "elasticsearch.cluster.id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))), "host":{ "name":body.name, "address":body.http_address, - } - }) - .with({ + }, "total":{ "flow":body.pipelines[pipeline_name].flow, "time":{ "queue_push_duration":{ - "ms":body.pipelines[pipeline_name]["events"]["queue_push_duration_in_millis"], + "ms":body.pipelines[pipeline_name].events.queue_push_duration_in_millis, }, "duration":{ - "ms":body.pipelines[pipeline_name]["events"]["duration_in_millis"], + "ms":body.pipelines[pipeline_name].events.duration_in_millis, }, }, "reloads":{ - "successes":body.pipelines[pipeline_name]["reloads"]["successes"], - "failures":body.pipelines[pipeline_name]["reloads"]["failures"] + "successes":body.pipelines[pipeline_name].reloads.successes, + "failures":body.pipelines[pipeline_name].reloads.failures }, "events":{ - "out":body.pipelines[pipeline_name]["events"]["out"], - "in":body.pipelines[pipeline_name]["events"]["in"], - "filtered":body.pipelines[pipeline_name]["events"]["filtered"], + "out":body.pipelines[pipeline_name].events.out, + "in":body.pipelines[pipeline_name].events["in"], + "filtered":body.pipelines[pipeline_name].events.filtered, }, "queues":{ - "type":body.pipelines[pipeline_name]["queue"]["type"], - "events":body.pipelines[pipeline_name]["queue"]["events_count"], + "type":body.pipelines[pipeline_name].queue["type"], + "events":body.pipelines[pipeline_name].queue.events_count, "current_size":{ - "bytes":body.pipelines[pipeline_name]["queue"]["queue_size_in_bytes"], + "bytes":body.pipelines[pipeline_name].queue.queue_size_in_bytes, }, "max_size":{ - "bytes":body.pipelines[pipeline_name]["queue"]["max_queue_size_in_bytes"], + "bytes":body.pipelines[pipeline_name].queue.max_queue_size_in_bytes, } } } @@ -66,7 +62,4 @@ program: | ) )) .as(pipelines, { - "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, - "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null - }, "events":pipelines.map(pipeline, {"logstash":{"pipeline":pipeline}})}) \ No newline at end of file diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index 96cba6d659e..459c91bcf21 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -14,8 +14,7 @@ auth.basic.password: {{escape_string password}} {{/if}} redact: - fields: - - password + fields: ~ program: | get(state.url) @@ -25,10 +24,10 @@ program: | .with({ "es_cluster_id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))), "es_cluster_id_map":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), {"plugin_id":each.id, "cluster_id":each.cluster_uuid}))), - "outputs":body.pipelines[pipeline_name]["plugins"]["outputs"], - "inputs":body.pipelines[pipeline_name]["plugins"]["inputs"], - "filters":body.pipelines[pipeline_name]["plugins"]["filters"], - "codecs":body.pipelines[pipeline_name]["plugins"]["codecs"], + "outputs":body.pipelines[pipeline_name].plugins.outputs, + "inputs":body.pipelines[pipeline_name].plugins.inputs, + "filters":body.pipelines[pipeline_name].plugins.filters, + "codecs":body.pipelines[pipeline_name].plugins.codecs, "host":{ "name":body.name, "address":body.http_address, @@ -46,7 +45,7 @@ program: | "plugin":{ "type":"input", "input":{ - "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == input.id), tuple.cluster_id), + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == input.id), tuple.cluster_id), "name":input.name, "id":input.id, "flow": has(input.flow) ? input.flow : {}, @@ -122,13 +121,13 @@ program: | "name":event.name, "id":event.hash, "host":event.host, - "elasticsearch.cluster.id":event['es_cluster_id'], + "elasticsearch.cluster.id":event.es_cluster_id, "plugin":{ "type":"output", "output":{ "id":output.id, "name":output.name, - "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == output.id), tuple.cluster_id), + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == output.id), tuple.cluster_id), "flow": has(output.flow) ? output.flow : {}, "events":{ "in":output.events['in'], @@ -144,7 +143,4 @@ program: | }.drop_empty() ) }).collate(["filters", "outputs", "inputs", "codecs"])).as(plugins, { - "cursor": {"username":(has(state.cursor) && has(state.cursor.username)) ? state.cursor.username : has(state.username) ? state.username : null, - "password":has(state.cursor) && has(state.cursor.password) ? state.cursor.password : has(state.password) ? state.password : null - }, "events":plugins.map(plugin, {"logstash":{"pipeline":plugin}})}) \ No newline at end of file From ffd380c5a866d23442346bd24b2d1c7b056f0fa9 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 6 Oct 2023 09:41:59 -0400 Subject: [PATCH 20/25] Style changes to cel scripts --- packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs | 2 +- packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs index 92010ea77e1..f77b6d8ea79 100644 --- a/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs @@ -48,7 +48,7 @@ program: | "filtered":body.pipelines[pipeline_name].events.filtered, }, "queues":{ - "type":body.pipelines[pipeline_name].queue["type"], + "type":body.pipelines[pipeline_name].queue.type, "events":body.pipelines[pipeline_name].queue.events_count, "current_size":{ "bytes":body.pipelines[pipeline_name].queue.queue_size_in_bytes, diff --git a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs index 459c91bcf21..2eaa7b40df1 100644 --- a/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/plugins/agent/stream/cel.yml.hbs @@ -101,7 +101,7 @@ program: | "filter":{ "id":filter.id, "name":filter.name, - "elasticsearch.cluster.id":event['es_cluster_id_map'].map(tuple, (tuple.plugin_id == filter.id), tuple.cluster_id), + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == filter.id), tuple.cluster_id), "flow": has(filter.flow) ? filter.flow : {}, "events":{ "in":filter.events['in'], From 7c2e47ea35b646d9105051fe6a89ee96ab30c73f Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 6 Oct 2023 10:50:25 -0400 Subject: [PATCH 21/25] Remove logstash monitoring pipeline from node view --- packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs index f289951d08e..359f20e2745 100644 --- a/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs +++ b/packages/logstash/data_stream/node_cel/agent/stream/cel.yml.hbs @@ -40,7 +40,7 @@ program: | "http_address":body.http_address, "name":body.name, "pipeline":body.pipeline, - "pipelines":body.pipelines.map(pipeline, [pipeline]).flatten(), + "pipelines":body.pipelines.map(pipeline, pipeline != '.monitoring-logstash', [pipeline]).flatten(), "snapshot":body.snapshot, "status":body.status, "uuid":body.id, From d47de6727ecdf0d63412bff27074d3c0b63b8907 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 6 Oct 2023 18:28:31 -0400 Subject: [PATCH 22/25] Polish dashboards --- ...-79270240-48ee-11ee-8cb5-99927777c522.json | 2 +- ...-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 24 +- ...-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 12 +- ...-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 4 +- ...-c0594170-526a-11ee-9ecc-31444cb79548.json | 4 +- ...-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 636 +++++++++++++++++- packages/logstash/manifest.yml | 4 +- 7 files changed, 656 insertions(+), 30 deletions(-) diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index fc966b0d1a3..82bfcb7f018 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1156,7 +1156,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:38:49.040Z", + "created_at": "2023-10-06T21:53:24.406Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index 08668a06609..aa90ad8b57c 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -4,7 +4,7 @@ "chainingSystem": "HIERARCHICAL", "controlStyle": "oneLine", "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"f8e0eae3-62e0-4539-87d6-523e89a8213e\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"f8e0eae3-62e0-4539-87d6-523e89a8213e\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"25b6ddfd-e372-4344-827b-c81461088031\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"25b6ddfd-e372-4344-827b-c81461088031\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" + "panelsJSON": "{\"f8e0eae3-62e0-4539-87d6-523e89a8213e\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"f8e0eae3-62e0-4539-87d6-523e89a8213e\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{},\"selectedOptions\":[]}},\"25b6ddfd-e372-4344-827b-c81461088031\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"25b6ddfd-e372-4344-827b-c81461088031\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" }, "description": "", "kibanaSavedObjectMeta": { @@ -184,7 +184,7 @@ "size": 1000 }, "scale": "ordinal", - "sourceField": "logstash.node.stats.logstash.name" + "sourceField": "logstash.host.name" }, "a51eda03-44a2-4173-be51-72948417a027": { "customLabel": true, @@ -1374,15 +1374,11 @@ "0c0ae432-64ed-4792-b539-c195c3d96549": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.pipeline.total.events.out: *" - }, "isBucketed": false, - "label": "Total events received", - "operationType": "last_value", + "label": "Total events emitted", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": true }, "scale": "ratio", "sourceField": "logstash.pipeline.total.events.out" @@ -1416,15 +1412,11 @@ "ca4b5da2-8007-4f89-8261-9081ee13cdd5": { "customLabel": true, "dataType": "number", - "filter": { - "language": "kuery", - "query": "logstash.pipeline.total.events.in: *" - }, "isBucketed": false, "label": "Total events received", - "operationType": "last_value", + "operationType": "max", "params": { - "sortField": "@timestamp" + "emptyAsNull": true }, "scale": "ratio", "sourceField": "logstash.pipeline.total.events.in" @@ -1570,7 +1562,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:38:49.040Z", + "created_at": "2023-10-06T21:53:24.406Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index bd14c21ada4..6a9bb66391e 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -132,10 +132,10 @@ "params": { "emptyAsNull": true, "format": { - "id": "number", + "id": "duration", "params": { "decimals": 0, - "suffix": "ms" + "fromUnit": "milliseconds" } } }, @@ -1208,6 +1208,7 @@ { "embeddableConfig": { "attributes": { + "description": "", "references": [ { "id": "logstash-sm-metrics", @@ -1228,13 +1229,14 @@ ], "columns": { "107f10cc-d65f-4a1a-9f54-ce90c48f4a5b": { + "customLabel": true, "dataType": "number", "filter": { "language": "kuery", "query": "logstash.pipeline.total.queues.events: *" }, "isBucketed": false, - "label": "Last value of logstash.pipeline.total.queues.events", + "label": "Number of events in queue", "operationType": "last_value", "params": { "format": { @@ -1897,9 +1899,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:38:49.040Z", + "created_at": "2023-10-06T22:09:20.002Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index 3e363461a81..dee3b6ad1ef 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -7030,9 +7030,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:52:49.964Z", + "created_at": "2023-10-06T21:53:24.406Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index 3f055fbfd21..ea9cf48c6e2 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -2002,9 +2002,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:55:00.707Z", + "created_at": "2023-10-06T21:53:24.406Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 43970a5ae2d..5d1f96a7357 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -51,7 +51,7 @@ } }, "gridData": { - "h": 24, + "h": 34, "i": "3175d525-4aa7-40b5-bc68-d89d105257de", "w": 8, "x": 0, @@ -902,6 +902,623 @@ "panelIndex": "4a093412-9812-433a-bf8d-225e4a402339", "type": "lens", "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { + "columnOrder": [ + "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "0c423bdf-578e-4f0a-bb27-180a24a133e1", + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "columns": { + "0c423bdf-578e-4f0a-bb27-180a24a133e1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.events.in : *" + }, + "isBucketed": false, + "label": "Events Emitted Rate", + "operationType": "counter_rate", + "references": [ + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "scale": "ratio", + "timeScale": "s" + }, + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.node.stats.events.out", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.out" + }, + "ef4e5445-487e-4a0c-ac01-063e8c199a84": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.stats.logstash.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": false, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "hideEndzones": true, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1" + ], + "collapseFn": "", + "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } + } + }, + "title": "Events Emitted Rate/s", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "acdc425f-0de2-46b6-8d9e-5dccdcb99270", + "w": 20, + "x": 8, + "y": 14 + }, + "panelIndex": "acdc425f-0de2-46b6-8d9e-5dccdcb99270", + "title": "Events Emitted Rate/s", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { + "columnOrder": [ + "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "0c423bdf-578e-4f0a-bb27-180a24a133e1", + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "columns": { + "0c423bdf-578e-4f0a-bb27-180a24a133e1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "logstash.node.stats.events.in : *" + }, + "isBucketed": false, + "label": "Events Received Rate", + "operationType": "counter_rate", + "references": [ + "cd7a8cba-084f-42b4-a4b5-334eee79e32e" + ], + "scale": "ratio", + "timeScale": "s" + }, + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "cd7a8cba-084f-42b4-a4b5-334eee79e32e": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of logstash.node.stats.events.in", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.in" + }, + "ef4e5445-487e-4a0c-ac01-063e8c199a84": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.stats.logstash.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": false, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "hideEndzones": true, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1" + ], + "collapseFn": "", + "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } + } + }, + "title": "Events Received Rate/s", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "fef0f0cc-45df-4389-9c13-78b56790905c", + "w": 20, + "x": 28, + "y": 14 + }, + "panelIndex": "fef0f0cc-45df-4389-9c13-78b56790905c", + "title": "Events Received Rate/s", + "type": "lens", + "version": "8.10.1" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logstash-sm-metrics", + "name": "indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425": { + "columnOrder": [ + "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "0c423bdf-578e-4f0a-bb27-180a24a133e1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" + ], + "columns": { + "0c423bdf-578e-4f0a-bb27-180a24a133e1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Event Latency (ms)", + "operationType": "formula", + "params": { + "formula": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", + "isFormulaBroken": false + }, + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4" + ], + "scale": "ratio" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.events.duration_in_millis\": *" + }, + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.duration_in_millis" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "counter_rate", + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.node.stats.events.out\": *" + }, + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.node.stats.events.out" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "counter_rate", + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X2" + ], + "scale": "ratio", + "timeScale": "s" + }, + "0c423bdf-578e-4f0a-bb27-180a24a133e1X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Event Latency (ms)", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" + ], + "location": { + "max": 128, + "min": 0 + }, + "name": "divide", + "text": "counter_rate(last_value(logstash.node.stats.events.duration_in_millis))/counter_rate(last_value(logstash.node.stats.events.out))", + "type": "function" + } + }, + "references": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1X1", + "0c423bdf-578e-4f0a-bb27-180a24a133e1X3" + ], + "scale": "ratio" + }, + "43c45a77-5bb2-4f66-8bfd-77f3cf386a87": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "ef4e5445-487e-4a0c-ac01-063e8c199a84": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.node.stats.logstash.name", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.stats.logstash.name" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": false, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "hideEndzones": true, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "0c423bdf-578e-4f0a-bb27-180a24a133e1" + ], + "collapseFn": "", + "layerId": "9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "ef4e5445-487e-4a0c-ac01-063e8c199a84", + "xAccessor": "43c45a77-5bb2-4f66-8bfd-77f3cf386a87", + "yConfig": [ + { + "color": "#ff0000", + "forAccessor": "0c423bdf-578e-4f0a-bb27-180a24a133e1" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "bottom", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + } + } + }, + "title": "Events Latency (ms) average", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "7bfdba6c-bcdc-4c51-be8c-41188c08dc6c", + "w": 20, + "x": 8, + "y": 24 + }, + "panelIndex": "7bfdba6c-bcdc-4c51-be8c-41188c08dc6c", + "title": "Events Latency (ms) average", + "type": "lens", + "version": "8.10.1" } ], "timeRestore": false, @@ -909,7 +1526,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-05T15:38:49.040Z", + "created_at": "2023-10-06T21:53:24.406Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", "managed": true, "references": [ @@ -943,6 +1560,21 @@ "name": "4a093412-9812-433a-bf8d-225e4a402339:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:1c4772e7-cf9a-417d-803f-3c7184b1ec82:dashboardId", "type": "dashboard" }, + { + "id": "logstash-sm-metrics", + "name": "acdc425f-0de2-46b6-8d9e-5dccdcb99270:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "fef0f0cc-45df-4389-9c13-78b56790905c:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + }, + { + "id": "logstash-sm-metrics", + "name": "7bfdba6c-bcdc-4c51-be8c-41188c08dc6c:indexpattern-datasource-layer-9f4942e8-bd51-41fe-9e6b-c6ca7ee81425", + "type": "index-pattern" + }, { "id": "logstash-sm-metrics", "name": "controlGroup_55797c81-115d-42f4-8b92-a90449de3183:optionsListDataView", diff --git a/packages/logstash/manifest.yml b/packages/logstash/manifest.yml index a494bd8aa5d..a6ce830cd2b 100644 --- a/packages/logstash/manifest.yml +++ b/packages/logstash/manifest.yml @@ -42,7 +42,7 @@ policy_templates: show_user: false - type: logstash/metrics title: "Metrics (Stack Monitoring)" - description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana. + description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana.\n Please disable if only using Technical Preview Logstash Monitoring" vars: - name: hosts @@ -86,7 +86,7 @@ policy_templates: required: false show_user: false - type: cel - title: "Metrics (Technical Preview). Please disable if using Stack Monitoring" + title: "Metrics (Technical Preview).\nPlease disable if using Stack Monitoring" description: "Collect Metrics and stats from Logstash instances to power dedicated Logstash Dashboards in Kibana" vars: From 637eda9df649da01a8edfe644736a9a6ca803afc Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Sat, 7 Oct 2023 16:34:39 -0400 Subject: [PATCH 23/25] Additional polish to dashboards --- .../logstash-79270240-48ee-11ee-8cb5-99927777c522.json | 6 +++--- .../logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 8 ++++---- .../logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 4 ++-- .../logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 8 ++++---- .../logstash-c0594170-526a-11ee-9ecc-31444cb79548.json | 8 ++++---- .../logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index 82bfcb7f018..2d512c0684c 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -20,7 +20,7 @@ "hidePanelTitles": false, "syncColors": true, "syncCursor": true, - "syncTooltips": false, + "syncTooltips": true, "useMargins": true }, "panelsJSON": [ @@ -1156,9 +1156,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T21:53:24.406Z", + "created_at": "2023-10-07T20:26:46.384Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index aa90ad8b57c..bf544605744 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -133,10 +133,10 @@ "params": { "emptyAsNull": true, "format": { - "id": "number", + "id": "duration", "params": { "decimals": 0, - "suffix": "ms" + "fromUnit": "milliseconds" } } }, @@ -1562,9 +1562,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T21:53:24.406Z", + "created_at": "2023-10-07T20:28:10.641Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index 6a9bb66391e..933382095ea 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -1899,9 +1899,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T22:09:20.002Z", + "created_at": "2023-10-07T20:10:23.644Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index dee3b6ad1ef..4d1049c3de1 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -18,9 +18,9 @@ }, "optionsJSON": { "hidePanelTitles": false, - "syncColors": false, + "syncColors": true, "syncCursor": true, - "syncTooltips": false, + "syncTooltips": true, "useMargins": true }, "panelsJSON": [ @@ -7030,9 +7030,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T21:53:24.406Z", + "created_at": "2023-10-07T20:29:04.038Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index ea9cf48c6e2..b9378f0c322 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -18,9 +18,9 @@ }, "optionsJSON": { "hidePanelTitles": false, - "syncColors": false, + "syncColors": true, "syncCursor": true, - "syncTooltips": false, + "syncTooltips": true, "useMargins": true }, "panelsJSON": [ @@ -2002,9 +2002,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T21:53:24.406Z", + "created_at": "2023-10-07T20:28:43.275Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 5d1f96a7357..1f8e05ec3c7 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -4,7 +4,7 @@ "chainingSystem": "HIERARCHICAL", "controlStyle": "oneLine", "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"55797c81-115d-42f4-8b92-a90449de3183\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"55797c81-115d-42f4-8b92-a90449de3183\",\"fieldName\":\"logstash.host.name\",\"title\":\"logstash.host.name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"logstash.pipeline.name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" + "panelsJSON": "{\"55797c81-115d-42f4-8b92-a90449de3183\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"55797c81-115d-42f4-8b92-a90449de3183\",\"fieldName\":\"logstash.host.name\",\"title\":\"Logstash Host Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}},\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"ffddbdfd-c666-4a7c-b81d-c66cb20212f4\",\"fieldName\":\"logstash.pipeline.name\",\"title\":\"Logstash Pipeline Name\",\"grow\":true,\"width\":\"medium\",\"enhancements\":{}}}}" }, "description": "", "kibanaSavedObjectMeta": { @@ -1526,9 +1526,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-06T21:53:24.406Z", + "created_at": "2023-10-07T20:27:23.468Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", - "managed": true, + "managed": false, "references": [ { "id": "logstash-sm-metrics", From 2e934fce97f57eb2e1d58068ccaeea15ae55c129 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 9 Oct 2023 09:33:35 -0400 Subject: [PATCH 24/25] Fix typo in nav panel, and fix flow metric to show correct plugin type in filter view --- .../logstash-79270240-48ee-11ee-8cb5-99927777c522.json | 4 ++-- .../logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json | 4 ++-- .../logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json | 2 +- .../logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json | 4 ++-- .../logstash-c0594170-526a-11ee-9ecc-31444cb79548.json | 4 ++-- .../logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json index 2d512c0684c..30e48eddb70 100644 --- a/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json +++ b/packages/logstash/kibana/dashboard/logstash-79270240-48ee-11ee-8cb5-99927777c522.json @@ -1156,9 +1156,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:26:46.384Z", + "created_at": "2023-10-09T13:22:35.298Z", "id": "logstash-79270240-48ee-11ee-8cb5-99927777c522", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json index bf544605744..4606c448229 100644 --- a/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json +++ b/packages/logstash/kibana/dashboard/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d.json @@ -1562,9 +1562,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:28:10.641Z", + "created_at": "2023-10-09T13:22:35.298Z", "id": "logstash-9d450b10-4680-11ee-9ddc-919f87fe352d", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json index 933382095ea..64784cb2865 100644 --- a/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json +++ b/packages/logstash/kibana/dashboard/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5.json @@ -1899,7 +1899,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:10:23.644Z", + "created_at": "2023-10-09T13:22:35.298Z", "id": "logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5", "managed": true, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json index 4d1049c3de1..4f899618b16 100644 --- a/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json +++ b/packages/logstash/kibana/dashboard/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc.json @@ -41,7 +41,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipeline Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n**[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)** \n\n\nOverview\n\nThis Dashboard gives a detailed view of a Logstash pipeline. \nUse controls to drill down on pipeline performance on each pipeline, and on individual nodes running those pipelines.", + "markdown": "**Navigation** \n\n**Logstash Overview**\n\n[Overview](/app/dashboards#/view/logstash-79270240-48ee-11ee-8cb5-99927777c522) \n[Nodes Overview](/app/dashboards#/view/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a) \n[Node Overview](/app/dashboards#/view/logstash-9d450b10-4680-11ee-9ddc-919f87fe352d) \n[Node Overview Advanced View](/app/dashboards#/view/logstash-a42d7060-45e6-11ee-957b-3720c0b0fbc5) \n\n[Pipelines Overview](/app/dashboards#/view/logstash-c0594170-526a-11ee-9ecc-31444cb79548) \n**[Pipeline Details Overview](/app/dashboards#/view/logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc)** \n\n\nOverview\n\nThis Dashboard gives a detailed view of a Logstash pipeline. \nUse controls to drill down on pipeline performance on each pipeline, and on individual nodes running those pipelines.", "openLinksInNewTab": false }, "title": "", @@ -7030,7 +7030,7 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:29:04.038Z", + "created_at": "2023-10-09T13:26:37.526Z", "id": "logstash-bc1a8050-5ee1-11ee-8e78-bf6865bc3ffc", "managed": false, "references": [ diff --git a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json index b9378f0c322..764ddf4b21d 100644 --- a/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json +++ b/packages/logstash/kibana/dashboard/logstash-c0594170-526a-11ee-9ecc-31444cb79548.json @@ -2002,9 +2002,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:28:43.275Z", + "created_at": "2023-10-09T13:22:35.298Z", "id": "logstash-c0594170-526a-11ee-9ecc-31444cb79548", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", diff --git a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json index 1f8e05ec3c7..74787d9fbb8 100644 --- a/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json +++ b/packages/logstash/kibana/dashboard/logstash-ee860840-41ed-11ee-874b-fdb94cc3273a.json @@ -1526,9 +1526,9 @@ "version": 1 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-10-07T20:27:23.468Z", + "created_at": "2023-10-09T13:22:35.298Z", "id": "logstash-ee860840-41ed-11ee-874b-fdb94cc3273a", - "managed": false, + "managed": true, "references": [ { "id": "logstash-sm-metrics", From 559a8ee393963037af30af7b36edb599f5f9b8f3 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 9 Oct 2023 11:31:50 -0400 Subject: [PATCH 25/25] remove duplicate fields Result of running: go run github.com/andrewkroh/fydler@main -fix packages/logstash/data_stream/*/fields/*yml --- .../data_stream/node_cel/fields/ecs.yml | 2 - .../data_stream/pipeline/fields/agent.yml | 117 ++++-------------- .../pipeline/fields/base-fields.yml | 9 +- .../data_stream/pipeline/fields/ecs.yml | 5 +- .../data_stream/pipeline/fields/fields.yml | 1 - .../data_stream/plugins/fields/agent.yml | 117 ++++-------------- .../plugins/fields/base-fields.yml | 9 +- .../data_stream/plugins/fields/ecs.yml | 5 +- .../data_stream/plugins/fields/fields.yml | 3 +- packages/logstash/docs/README.md | 36 +++--- packages/logstash/validation.yml | 3 - 11 files changed, 77 insertions(+), 230 deletions(-) delete mode 100644 packages/logstash/validation.yml diff --git a/packages/logstash/data_stream/node_cel/fields/ecs.yml b/packages/logstash/data_stream/node_cel/fields/ecs.yml index d9087c71721..29b7c409f04 100644 --- a/packages/logstash/data_stream/node_cel/fields/ecs.yml +++ b/packages/logstash/data_stream/node_cel/fields/ecs.yml @@ -20,7 +20,5 @@ external: ecs - name: event.module external: ecs -- name: host.name - external: ecs - name: error.message external: ecs diff --git a/packages/logstash/data_stream/pipeline/fields/agent.yml b/packages/logstash/data_stream/pipeline/fields/agent.yml index d0032fb9fff..19f18761f64 100644 --- a/packages/logstash/data_stream/pipeline/fields/agent.yml +++ b/packages/logstash/data_stream/pipeline/fields/agent.yml @@ -1,141 +1,74 @@ - name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - name: account.id - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + external: ecs - name: availability_zone - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. + external: ecs - name: instance.id - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. + external: ecs - name: instance.name - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. + external: ecs - name: machine.type - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. + external: ecs - name: provider - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + external: ecs - name: region - type: keyword - ignore_above: 1024 - description: Region in which this host is running. + external: ecs - name: project.id - type: keyword - description: Name of the project in Google Cloud. + external: ecs - name: image.id type: keyword description: Image ID for the cloud instance. - name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." type: group fields: - name: id - type: keyword - ignore_above: 1024 - description: Unique container id. + external: ecs - name: image.name - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. + external: ecs - name: labels - type: object - object_type: keyword - description: Image labels. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: Container name. + external: ecs - name: host - title: Host - group: 2 - description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - name: architecture - type: keyword - ignore_above: 1024 - description: Operating system architecture. + external: ecs - name: domain - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - default_field: false + external: ecs - name: hostname - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + external: ecs - name: id - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + external: ecs - name: ip - type: ip - description: Host ip addresses. + external: ecs - name: mac - type: keyword - ignore_above: 1024 - description: Host mac addresses. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + external: ecs - name: os.family - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). + external: ecs - name: os.kernel - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. + external: ecs - name: os.name - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. + external: ecs - name: os.platform - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). + external: ecs - name: os.version - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. + external: ecs - name: type - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + external: ecs - name: containerized type: boolean description: > If the host is a container. - - name: os.build type: keyword description: > OS build information. - - name: os.codename type: keyword description: > OS codename, if any. - diff --git a/packages/logstash/data_stream/pipeline/fields/base-fields.yml b/packages/logstash/data_stream/pipeline/fields/base-fields.yml index 4227333a0d2..1bfb71226cf 100644 --- a/packages/logstash/data_stream/pipeline/fields/base-fields.yml +++ b/packages/logstash/data_stream/pipeline/fields/base-fields.yml @@ -1,12 +1,9 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace - type: constant_keyword - description: Data stream namespace. + external: ecs - name: service.hostname type: keyword description: Hostname of the service diff --git a/packages/logstash/data_stream/pipeline/fields/ecs.yml b/packages/logstash/data_stream/pipeline/fields/ecs.yml index aad92fac0ef..29b7c409f04 100644 --- a/packages/logstash/data_stream/pipeline/fields/ecs.yml +++ b/packages/logstash/data_stream/pipeline/fields/ecs.yml @@ -7,8 +7,7 @@ - name: service.version external: ecs - name: service.address - type: keyword - description: Address where data about this service was collected from. + external: ecs - name: service.name external: ecs - name: process.pid @@ -21,7 +20,5 @@ external: ecs - name: event.module external: ecs -- name: host.name - external: ecs - name: error.message external: ecs diff --git a/packages/logstash/data_stream/pipeline/fields/fields.yml b/packages/logstash/data_stream/pipeline/fields/fields.yml index 6ce6b92db9e..de807ffc36e 100644 --- a/packages/logstash/data_stream/pipeline/fields/fields.yml +++ b/packages/logstash/data_stream/pipeline/fields/fields.yml @@ -1,6 +1,5 @@ - name: logstash.pipeline type: group - description: Logstash pipeline metrics fields: - name: name type: keyword diff --git a/packages/logstash/data_stream/plugins/fields/agent.yml b/packages/logstash/data_stream/plugins/fields/agent.yml index d0032fb9fff..19f18761f64 100644 --- a/packages/logstash/data_stream/plugins/fields/agent.yml +++ b/packages/logstash/data_stream/plugins/fields/agent.yml @@ -1,141 +1,74 @@ - name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - name: account.id - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + external: ecs - name: availability_zone - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. + external: ecs - name: instance.id - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. + external: ecs - name: instance.name - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. + external: ecs - name: machine.type - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. + external: ecs - name: provider - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + external: ecs - name: region - type: keyword - ignore_above: 1024 - description: Region in which this host is running. + external: ecs - name: project.id - type: keyword - description: Name of the project in Google Cloud. + external: ecs - name: image.id type: keyword description: Image ID for the cloud instance. - name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." type: group fields: - name: id - type: keyword - ignore_above: 1024 - description: Unique container id. + external: ecs - name: image.name - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. + external: ecs - name: labels - type: object - object_type: keyword - description: Image labels. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: Container name. + external: ecs - name: host - title: Host - group: 2 - description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - name: architecture - type: keyword - ignore_above: 1024 - description: Operating system architecture. + external: ecs - name: domain - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - default_field: false + external: ecs - name: hostname - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + external: ecs - name: id - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + external: ecs - name: ip - type: ip - description: Host ip addresses. + external: ecs - name: mac - type: keyword - ignore_above: 1024 - description: Host mac addresses. + external: ecs - name: name - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + external: ecs - name: os.family - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). + external: ecs - name: os.kernel - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. + external: ecs - name: os.name - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. + external: ecs - name: os.platform - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). + external: ecs - name: os.version - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. + external: ecs - name: type - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + external: ecs - name: containerized type: boolean description: > If the host is a container. - - name: os.build type: keyword description: > OS build information. - - name: os.codename type: keyword description: > OS codename, if any. - diff --git a/packages/logstash/data_stream/plugins/fields/base-fields.yml b/packages/logstash/data_stream/plugins/fields/base-fields.yml index 4227333a0d2..1bfb71226cf 100644 --- a/packages/logstash/data_stream/plugins/fields/base-fields.yml +++ b/packages/logstash/data_stream/plugins/fields/base-fields.yml @@ -1,12 +1,9 @@ - name: data_stream.type - type: constant_keyword - description: Data stream type. + external: ecs - name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. + external: ecs - name: data_stream.namespace - type: constant_keyword - description: Data stream namespace. + external: ecs - name: service.hostname type: keyword description: Hostname of the service diff --git a/packages/logstash/data_stream/plugins/fields/ecs.yml b/packages/logstash/data_stream/plugins/fields/ecs.yml index aad92fac0ef..29b7c409f04 100644 --- a/packages/logstash/data_stream/plugins/fields/ecs.yml +++ b/packages/logstash/data_stream/plugins/fields/ecs.yml @@ -7,8 +7,7 @@ - name: service.version external: ecs - name: service.address - type: keyword - description: Address where data about this service was collected from. + external: ecs - name: service.name external: ecs - name: process.pid @@ -21,7 +20,5 @@ external: ecs - name: event.module external: ecs -- name: host.name - external: ecs - name: error.message external: ecs diff --git a/packages/logstash/data_stream/plugins/fields/fields.yml b/packages/logstash/data_stream/plugins/fields/fields.yml index e898500dd4f..8fbb6608f89 100644 --- a/packages/logstash/data_stream/plugins/fields/fields.yml +++ b/packages/logstash/data_stream/plugins/fields/fields.yml @@ -1,6 +1,5 @@ - name: logstash.pipeline type: group - description: Logstash pipeline metrics fields: - name: name type: keyword @@ -190,4 +189,4 @@ - name: worker_utilization.last_1_minute type: scaled_float metric_type: gauge - description: worker utilization for this plugin \ No newline at end of file + description: worker utilization for this plugin diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 9acc3e101b1..4502a98fd4a 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -835,22 +835,22 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages. |---|---|---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | | cloud.instance.id | Instance ID of the host machine. | keyword | | | | cloud.instance.name | Instance name of the host machine. | keyword | | | | cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | -| cloud.region | Region in which this host is running. | keyword | | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | | | cluster_uuid | | alias | | | | container.id | Unique container id. | keyword | | | | container.image.name | Name of the image the container was built on. | keyword | | | | container.labels | Image labels. | object | | | | container.name | Container name. | keyword | | | -| data_stream.dataset | Data stream dataset. | constant_keyword | | | -| data_stream.namespace | Data stream namespace. | constant_keyword | | | -| data_stream.type | Data stream type. | constant_keyword | | | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | error.message | Error message. | match_only_text | | | | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | @@ -862,14 +862,14 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages. | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | | host.ip | Host ip addresses. | ip | | | -| host.mac | Host mac addresses. | keyword | | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | | host.os.kernel | Operating system kernel version as a raw string. | keyword | | | | host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | | host.os.version | Operating system version as a raw string. | keyword | | | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | @@ -906,7 +906,7 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages. | logstash.pipeline.total.time.duration.ms | Time spent processing events through the pipeline. | long | ms | counter | | logstash.pipeline.total.time.queue_push_duration.ms | Time spent pushing events to the queue for this pipeline. | long | ms | counter | | process.pid | Process id. | long | | | -| service.address | Address where data about this service was collected from. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | | service.hostname | Hostname of the service | keyword | | | | service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | | service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | @@ -1031,22 +1031,22 @@ pipeline collection period, and setting it to an appropriate value. |---|---|---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | | cloud.instance.id | Instance ID of the host machine. | keyword | | | | cloud.instance.name | Instance name of the host machine. | keyword | | | | cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | -| cloud.region | Region in which this host is running. | keyword | | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | | | cluster_uuid | | alias | | | | container.id | Unique container id. | keyword | | | | container.image.name | Name of the image the container was built on. | keyword | | | | container.labels | Image labels. | object | | | | container.name | Container name. | keyword | | | -| data_stream.dataset | Data stream dataset. | constant_keyword | | | -| data_stream.namespace | Data stream namespace. | constant_keyword | | | -| data_stream.type | Data stream type. | constant_keyword | | | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | error.message | Error message. | match_only_text | | | | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | @@ -1058,14 +1058,14 @@ pipeline collection period, and setting it to an appropriate value. | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | | host.ip | Host ip addresses. | ip | | | -| host.mac | Host mac addresses. | keyword | | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | | host.os.kernel | Operating system kernel version as a raw string. | keyword | | | | host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | | host.os.version | Operating system version as a raw string. | keyword | | | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | @@ -1113,7 +1113,7 @@ pipeline collection period, and setting it to an appropriate value. | logstash.pipeline.plugin.output.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | | logstash.pipeline.plugin.type | Type of the plugin | keyword | | | | process.pid | Process id. | long | | | -| service.address | Address where data about this service was collected from. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | | service.hostname | Hostname of the service | keyword | | | | service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | | service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | diff --git a/packages/logstash/validation.yml b/packages/logstash/validation.yml deleted file mode 100644 index 2527f20c354..00000000000 --- a/packages/logstash/validation.yml +++ /dev/null @@ -1,3 +0,0 @@ -errors: - exclude_checks: - - SVR00002 # Mandatory filters in dashboards.