From 707b05724f762b4396c91771a01605f3a6dce99c Mon Sep 17 00:00:00 2001 From: rohit-bandlamudi-nr Date: Tue, 10 Dec 2024 16:53:11 +0530 Subject: [PATCH 01/12] feat: add entity synthesis rules to fluenbit entity --- entity-types/ext-fluentbit/definition.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/entity-types/ext-fluentbit/definition.yml b/entity-types/ext-fluentbit/definition.yml index 65436e940..793bd46bf 100644 --- a/entity-types/ext-fluentbit/definition.yml +++ b/entity-types/ext-fluentbit/definition.yml @@ -1,2 +1,16 @@ domain: EXT -type: FLUENTBIT \ No newline at end of file +type: FLUENTBIT +synthesis: + rules: + - compositeIdentifier: + separator: "/" + attributes: + - namespace + - helm_release_name + name: helm_release_name + encodeIdentifierInGUID: true + conditions: + - attribute: eventType + value: Metric + - attribute: metricName + prefix: fluentbit_ \ No newline at end of file From ef5a8b5b63748e9b7a765ceb9abaaca32bbf7722 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 17:59:40 +0530 Subject: [PATCH 02/12] included dashboard for flunet bit --- entity-types/ext-fluentbit/dashboard.json | 2139 +++++++++++++++++++++ 1 file changed, 2139 insertions(+) create mode 100644 entity-types/ext-fluentbit/dashboard.json diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json new file mode 100644 index 000000000..dc9d5460a --- /dev/null +++ b/entity-types/ext-fluentbit/dashboard.json @@ -0,0 +1,2139 @@ +{ + "name": "Kubernetes Fluent Bit monitoring", + "description": null, + "permissions": "PUBLIC_READ_WRITE", + "pages": [ + { + "name": "Fluent Bit metrics: General", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 6 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" + } + }, + { + "title": "Fluent Bit version", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Fluent Bit uptime", + "layout": { + "column": 7, + "row": 4, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 7, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# INPUTS" + } + }, + { + "title": "Input byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Input log rate (records/minute)", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average incoming record size (bytes)", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 11, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# FILTERS" + } + }, + { + "title": "Filter byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filter log rate (records/minute)", + "layout": { + "column": 5, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average filtered record size (bytes)", + "layout": { + "column": 9, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Record add/drop rate per FILTER plugin", + "layout": { + "column": 1, + "row": 15, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "REQUESTS_PER_MINUTE" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 18, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# OUTPUTS" + } + }, + { + "title": "Output byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Output log rate (records/minute)", + "layout": { + "column": 5, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average outgoing record size (bytes)", + "layout": { + "column": 9, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin statistics (records/minute)", + "layout": { + "column": 1, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Other OUTPUT plugin statistics (records/minute)", + "layout": { + "column": 5, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'newrelic-logs-forwarder' and name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Connections per OUTPUT plugin", + "layout": { + "column": 9, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_output_upstream_total_connections) as 'Total', max(fluentbit_output_upstream_busy_connections) as 'Busy' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin errors (errors/minute)", + "layout": { + "column": 1, + "row": 25, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin chunk retry statistics (retries/minute)", + "layout": { + "column": 5, + "row": 25, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 28, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# MEMORY USAGE" + } + }, + { + "title": "Input plugin memory usage", + "layout": { + "column": 1, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "INPUT memory buffer over limit", + "layout": { + "column": 5, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#013ef4", + "seriesName": "pod-logs-tailer" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true, + "thresholds": [ + { + "from": 0.95, + "name": "Mem buf overlimit", + "severity": "critical", + "to": 1.05 + } + ] + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Chunk statistics per INPUT plugin", + "layout": { + "column": 9, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Buffered chunks", + "layout": { + "column": 1, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_chunks) AS 'Total', max(fluentbit_storage_mem_chunks) AS 'Memory', max(fluentbit_storage_fs_chunks) AS 'Filesystem' FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Busy chunks' size", + "layout": { + "column": 5, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filesystem chunks state", + "layout": { + "column": 9, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(fluentbit_storage_fs_chunks_up) AS 'Up (in memory)', average(fluentbit_storage_fs_chunks_down) AS 'Down (fs only)' FROM Metric since '2024-02-29 13:22:00+0000' UNTIL '2024-02-29 14:31:00+0000' timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + }, + { + "name": "Fluent Bit metrics: Pipeline View", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 6 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# README\n\n## About this page\nThis page represents the same metrics that are displayed in the \"Fluent Bit metrics: General\" page. Nevertheless, they are grouped differently to allow you to visualize a given metric across the whole pipeline with a single glance.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" + } + }, + { + "title": "Fluent Bit version", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Fluent Bit uptime", + "layout": { + "column": 7, + "row": 4, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) timeseries facet pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "SECONDS" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 7, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# BYTE RATES" + } + }, + { + "title": "Input byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filter byte rate (bytes/minute)", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Output byte rate (bytes/minute)", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 11, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# LOG RECORD RATES" + } + }, + { + "title": "Input log rate (records/minute)", + "layout": { + "column": 1, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filter log rate (records/minute)", + "layout": { + "column": 5, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Output log rate (records/minute)", + "layout": { + "column": 9, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Record add/drop rate per FILTER plugin", + "layout": { + "column": 5, + "row": 15, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "REQUESTS_PER_MINUTE" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 18, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# AVERAGE LOG RECORD SIZES AT THE END OF EACH STAGE" + } + }, + { + "title": "Average incoming record size (bytes)", + "layout": { + "column": 1, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average filtered record size (bytes)", + "layout": { + "column": 5, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average outgoing record size (bytes)", + "layout": { + "column": 9, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 22, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# MEMORY USAGE AND BACKPRESSURE" + } + }, + { + "title": "Input plugin memory usage", + "layout": { + "column": 1, + "row": 23, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Busy chunks' size", + "layout": { + "column": 5, + "row": 23, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin chunk retry statistics (retries/minute)", + "layout": { + "column": 9, + "row": 23, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "INPUT memory buffer over limit", + "layout": { + "column": 1, + "row": 26, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#013ef4", + "seriesName": "pod-logs-tailer" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true, + "thresholds": [ + { + "from": 0.95, + "name": "Mem buf overlimit", + "severity": "critical", + "to": 1.05 + } + ] + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Chunk statistics per INPUT plugin", + "layout": { + "column": 5, + "row": 26, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin errors (errors/minute)", + "layout": { + "column": 9, + "row": 26, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + }, + { + "name": "newrelic-fluent-bit-output plugin metrics", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 4, + "height": 9 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# README\n## About this page\nThis page displays metrics collected internally in the [New Relic Fluent Bit output plugin](https://github.com/newrelic/newrelic-fluent-bit-output) (in short, **NR FB plugin**). These metrics are independent of Fluent Bit's, and **must not be considered as a stable API: they can change its naming or dimensions at any time in newer plugin versions**.\n\nPlease note that **the NR FB plugin does not include the `pod_name` nor the `node_name` dimensions**. Therefore, the graphs below represent an aggregation of all your running Fluent Bit instances across one or more clusters. You can use the `cluster_name` dimension (or dashboard variable above) to narrow down the troubleshooting to one or more clusters.\n\n## Basic naming conventions\n- Fluent Bit aggregates logs in batches, also referred as **[chunks](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure)**. Each chunk therefore contains an unknown amount of logs.\n- Chunks are received sequentially at the NR FB plugin, which takes care of reading the logs they contain and splitting them into the so-called New Relic *payloads*.\n- Each **payload** is a compressed stream of bytes that can be [at most 1MB long](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#limits), and follows the [data format required by the Logs API](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#json-content).\n\n\n## Error-detection graphs and recommended actions\n\nThe following are the main graphs used to detect potential problems in your log forwarding setup. Refer to each section to learn the recommended actions for each graph.\n\n### Payload packaging errors\nRepresents the percentage of Fluent Bit chunks that threw an error when they were attempted to be packaged as New Relic payloads. Such errors are never expected to happen. Therefore, **any value greater than 0% should be thoroughly investigated**.\n\nIf you find errors in this graph, please open a support ticket and include a sample of your logs for further investigation.\n\n### Payload sending errors\nRepresents the percentage of New Relic payloads that threw an unexpected error when they were attempted to be sent to New Relic. Such errors can happen sporadically: timeouts due to poor network performance or sudden network changes can cause them from time to time. Observing **values greater than 0% can sometimes be normal, but any value above 10% should be considered as an annomalous situation and should be thoroughly investigated**.\n\nIf you find errors in this graph, please ensure that you don't have any weak spots in your network path to New Relic: are you using a proxy? Is it or any network hop introducing too much latency due to being saturated? If you can't find anything on you side, please open a support ticket and include as much information as possible from your network setup.\n\n### Payload send results\nRepresents the amount of API requests that were performed to send logs to New Relic. **Ideally, you should only observe 202 responses here**. Sometimes, intermediary CDN providers can introduce some errors (503 error codes) from time to time, in which case your logs will not be lost and reattempted to be sent.\n\nIf you find a considerable amount of non-202 responses in this graph, please open a customer support ticket.\n\n## Additional troubleshooting graphs\n\nThe following graphs include additional fine-grained information that will be useful for New Relic to troubleshoot your potential installation issues.\n\n### Average timings\nRepresents the average amount of time the plugin spent packaging the log payloads and sending them to New Relic, respectively.\n\n### Accumulated time per minute\nRepresents the amount of time per minute the plugin spent packaging the log payloads and sending them to New Relic, respectively.\n\n### Payload size\nRepresents the size in bytes of the individual compressed payloads sent to New Relic.\n\n### Payload packets per Fluent Bit chunk\nRepresents the amount of payloads sent to New Relic per each Fluent Bit chunk." + } + }, + { + "title": "Payload packaging errors", + "layout": { + "column": 5, + "row": 1, + "width": 2, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT percentage(count(`logs.fb.packaging.time`), WHERE hasError = true) AS 'packaging errors'" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [ + { + "alertSeverity": "CRITICAL", + "value": 0 + } + ] + } + }, + { + "title": "Payload sending errors", + "layout": { + "column": 7, + "row": 1, + "width": 2, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT percentage(count(`logs.fb.payload.send.time`), WHERE hasError = true) AS 'send errors'" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [ + { + "alertSeverity": "WARNING", + "value": 0 + }, + { + "alertSeverity": "CRITICAL", + "value": 0.1 + } + ] + } + }, + { + "title": "Payload send results", + "layout": { + "column": 9, + "row": 1, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(count(logs.fb.payload.send.time), 1 minute) AS 'Status Code' FROM Metric FACET CASES(WHERE statusCode = 0 AS 'Send error') OR statusCode timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "REQUESTS_PER_MINUTE" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average timings", + "layout": { + "column": 5, + "row": 4, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(logs.fb.payload.send.time) AS 'Payload sending', average(logs.fb.packaging.time) AS 'Payload packaging' FROM Metric timeseries max" + } + ], + "nullValues": { + "nullValue": "zero" + }, + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "MS" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Accumulated time per minute", + "layout": { + "column": 9, + "row": 4, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(logs.fb.total.send.time), 1 minute) AS 'Sending', rate(sum(logs.fb.packaging.time), 1 minute) AS 'Packaging' FROM Metric TIMESERIES max" + } + ], + "nullValues": { + "nullValue": "zero" + }, + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "MS" + } + } + }, + { + "title": "Payload size", + "layout": { + "column": 5, + "row": 7, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT min(logs.fb.payload.size) AS 'Minimum', average(logs.fb.payload.size) AS 'Average', max(logs.fb.payload.size) AS 'Maximum' FROM Metric timeseries MAX " + } + ], + "nullValues": { + "nullValue": "default" + }, + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Payload packets per Fluent Bit chunk", + "layout": { + "column": 9, + "row": 7, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT min(logs.fb.payload.count) AS 'Minimum', average(logs.fb.payload.count) AS 'Average', max(logs.fb.payload.count) AS 'Maximum' FROM Metric timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "COUNT" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } + ], + "variables": [ + { + "name": "cluster_name", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountId": 0, + "query": "SELECT uniques(cluster_name) FROM Metric where metricName = 'fluentbit_input_storage_overlimit'" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Cluster Name", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + } + ] +} \ No newline at end of file From a7d7da40a8119583d1f2a3c872681dbc0c39ec9f Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 18:40:37 +0530 Subject: [PATCH 03/12] feat: updated dashboard.json of FB --- entity-types/ext-fluentbit/dashboard.json | 3074 +++++++-------------- 1 file changed, 947 insertions(+), 2127 deletions(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index dc9d5460a..335769007 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -1,2139 +1,959 @@ { "name": "Kubernetes Fluent Bit monitoring", "description": null, - "permissions": "PUBLIC_READ_WRITE", "pages": [ - { - "name": "Fluent Bit metrics: General", - "description": null, - "widgets": [ - { - "title": "", - "layout": { - "column": 1, - "row": 1, - "width": 6, - "height": 6 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" - } - }, - { - "title": "Fluent Bit version", - "layout": { - "column": 7, - "row": 1, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.table" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Fluent Bit uptime", - "layout": { - "column": 7, - "row": 4, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 7, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# INPUTS" - } - }, - { - "title": "Input byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Input log rate (records/minute)", - "layout": { - "column": 5, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average incoming record size (bytes)", - "layout": { - "column": 9, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 11, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# FILTERS" - } - }, - { - "title": "Filter byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Filter log rate (records/minute)", - "layout": { - "column": 5, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average filtered record size (bytes)", - "layout": { - "column": 9, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Record add/drop rate per FILTER plugin", - "layout": { - "column": 1, - "row": 15, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "REQUESTS_PER_MINUTE" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 18, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# OUTPUTS" - } - }, - { - "title": "Output byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Output log rate (records/minute)", - "layout": { - "column": 5, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average outgoing record size (bytes)", - "layout": { - "column": 9, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin statistics (records/minute)", - "layout": { - "column": 1, - "row": 22, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Other OUTPUT plugin statistics (records/minute)", - "layout": { - "column": 5, - "row": 22, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'newrelic-logs-forwarder' and name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Connections per OUTPUT plugin", - "layout": { - "column": 9, - "row": 22, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_output_upstream_total_connections) as 'Total', max(fluentbit_output_upstream_busy_connections) as 'Busy' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin errors (errors/minute)", - "layout": { - "column": 1, - "row": 25, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin chunk retry statistics (retries/minute)", - "layout": { - "column": 5, - "row": 25, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 28, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# MEMORY USAGE" - } - }, - { - "title": "Input plugin memory usage", - "layout": { - "column": 1, - "row": 29, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "INPUT memory buffer over limit", - "layout": { - "column": 5, - "row": 29, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "colors": { - "seriesOverrides": [ - { - "color": "#013ef4", - "seriesName": "pod-logs-tailer" - } - ] - }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true, - "thresholds": [ - { - "from": 0.95, - "name": "Mem buf overlimit", - "severity": "critical", - "to": 1.05 - } - ] - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Chunk statistics per INPUT plugin", - "layout": { - "column": 9, - "row": 29, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Buffered chunks", - "layout": { - "column": 1, - "row": 32, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_chunks) AS 'Total', max(fluentbit_storage_mem_chunks) AS 'Memory', max(fluentbit_storage_fs_chunks) AS 'Filesystem' FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Busy chunks' size", - "layout": { - "column": 5, - "row": 32, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Filesystem chunks state", - "layout": { - "column": 9, - "row": 32, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT average(fluentbit_storage_fs_chunks_up) AS 'Up (in memory)', average(fluentbit_storage_fs_chunks_down) AS 'Down (fs only)' FROM Metric since '2024-02-29 13:22:00+0000' UNTIL '2024-02-29 14:31:00+0000' timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } + { + "name": "Fluent Bit metrics: General", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 6 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" + } + }, + { + "title": "Fluent Bit version", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" } - ] - }, - { - "name": "Fluent Bit metrics: Pipeline View", - "description": null, - "widgets": [ - { - "title": "", - "layout": { - "column": 1, - "row": 1, - "width": 6, - "height": 6 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# README\n\n## About this page\nThis page represents the same metrics that are displayed in the \"Fluent Bit metrics: General\" page. Nevertheless, they are grouped differently to allow you to visualize a given metric across the whole pipeline with a single glance.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" - } - }, - { - "title": "Fluent Bit version", - "layout": { - "column": 7, - "row": 1, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.table" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Fluent Bit uptime", - "layout": { - "column": 7, - "row": 4, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) timeseries facet pod_name " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "SECONDS" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 7, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# BYTE RATES" - } - }, - { - "title": "Input byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Filter byte rate (bytes/minute)", - "layout": { - "column": 5, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Output byte rate (bytes/minute)", - "layout": { - "column": 9, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 11, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# LOG RECORD RATES" - } - }, - { - "title": "Input log rate (records/minute)", - "layout": { - "column": 1, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Filter log rate (records/minute)", - "layout": { - "column": 5, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Output log rate (records/minute)", - "layout": { - "column": 9, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Record add/drop rate per FILTER plugin", - "layout": { - "column": 5, - "row": 15, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "REQUESTS_PER_MINUTE" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 18, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# AVERAGE LOG RECORD SIZES AT THE END OF EACH STAGE" - } - }, - { - "title": "Average incoming record size (bytes)", - "layout": { - "column": 1, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average filtered record size (bytes)", - "layout": { - "column": 5, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average outgoing record size (bytes)", - "layout": { - "column": 9, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 22, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# MEMORY USAGE AND BACKPRESSURE" - } - }, - { - "title": "Input plugin memory usage", - "layout": { - "column": 1, - "row": 23, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Busy chunks' size", - "layout": { - "column": 5, - "row": 23, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin chunk retry statistics (retries/minute)", - "layout": { - "column": 9, - "row": 23, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "INPUT memory buffer over limit", - "layout": { - "column": 1, - "row": 26, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "colors": { - "seriesOverrides": [ - { - "color": "#013ef4", - "seriesName": "pod-logs-tailer" - } - ] - }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true, - "thresholds": [ - { - "from": 0.95, - "name": "Mem buf overlimit", - "severity": "critical", - "to": 1.05 - } - ] - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Chunk statistics per INPUT plugin", - "layout": { - "column": 5, - "row": 26, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin errors (errors/minute)", - "layout": { - "column": 9, - "row": 26, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Fluent Bit uptime", + "layout": { + "column": 7, + "row": 4, + "width": 6, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries" } - ] - }, - { - "name": "newrelic-fluent-bit-output plugin metrics", - "description": null, - "widgets": [ - { - "title": "", - "layout": { - "column": 1, - "row": 1, - "width": 4, - "height": 9 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# README\n## About this page\nThis page displays metrics collected internally in the [New Relic Fluent Bit output plugin](https://github.com/newrelic/newrelic-fluent-bit-output) (in short, **NR FB plugin**). These metrics are independent of Fluent Bit's, and **must not be considered as a stable API: they can change its naming or dimensions at any time in newer plugin versions**.\n\nPlease note that **the NR FB plugin does not include the `pod_name` nor the `node_name` dimensions**. Therefore, the graphs below represent an aggregation of all your running Fluent Bit instances across one or more clusters. You can use the `cluster_name` dimension (or dashboard variable above) to narrow down the troubleshooting to one or more clusters.\n\n## Basic naming conventions\n- Fluent Bit aggregates logs in batches, also referred as **[chunks](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure)**. Each chunk therefore contains an unknown amount of logs.\n- Chunks are received sequentially at the NR FB plugin, which takes care of reading the logs they contain and splitting them into the so-called New Relic *payloads*.\n- Each **payload** is a compressed stream of bytes that can be [at most 1MB long](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#limits), and follows the [data format required by the Logs API](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#json-content).\n\n\n## Error-detection graphs and recommended actions\n\nThe following are the main graphs used to detect potential problems in your log forwarding setup. Refer to each section to learn the recommended actions for each graph.\n\n### Payload packaging errors\nRepresents the percentage of Fluent Bit chunks that threw an error when they were attempted to be packaged as New Relic payloads. Such errors are never expected to happen. Therefore, **any value greater than 0% should be thoroughly investigated**.\n\nIf you find errors in this graph, please open a support ticket and include a sample of your logs for further investigation.\n\n### Payload sending errors\nRepresents the percentage of New Relic payloads that threw an unexpected error when they were attempted to be sent to New Relic. Such errors can happen sporadically: timeouts due to poor network performance or sudden network changes can cause them from time to time. Observing **values greater than 0% can sometimes be normal, but any value above 10% should be considered as an annomalous situation and should be thoroughly investigated**.\n\nIf you find errors in this graph, please ensure that you don't have any weak spots in your network path to New Relic: are you using a proxy? Is it or any network hop introducing too much latency due to being saturated? If you can't find anything on you side, please open a support ticket and include as much information as possible from your network setup.\n\n### Payload send results\nRepresents the amount of API requests that were performed to send logs to New Relic. **Ideally, you should only observe 202 responses here**. Sometimes, intermediary CDN providers can introduce some errors (503 error codes) from time to time, in which case your logs will not be lost and reattempted to be sent.\n\nIf you find a considerable amount of non-202 responses in this graph, please open a customer support ticket.\n\n## Additional troubleshooting graphs\n\nThe following graphs include additional fine-grained information that will be useful for New Relic to troubleshoot your potential installation issues.\n\n### Average timings\nRepresents the average amount of time the plugin spent packaging the log payloads and sending them to New Relic, respectively.\n\n### Accumulated time per minute\nRepresents the amount of time per minute the plugin spent packaging the log payloads and sending them to New Relic, respectively.\n\n### Payload size\nRepresents the size in bytes of the individual compressed payloads sent to New Relic.\n\n### Payload packets per Fluent Bit chunk\nRepresents the amount of payloads sent to New Relic per each Fluent Bit chunk." - } - }, - { - "title": "Payload packaging errors", - "layout": { - "column": 5, - "row": 1, - "width": 2, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.billboard" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT percentage(count(`logs.fb.packaging.time`), WHERE hasError = true) AS 'packaging errors'" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [ - { - "alertSeverity": "CRITICAL", - "value": 0 - } - ] - } - }, - { - "title": "Payload sending errors", - "layout": { - "column": 7, - "row": 1, - "width": 2, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.billboard" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT percentage(count(`logs.fb.payload.send.time`), WHERE hasError = true) AS 'send errors'" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [ - { - "alertSeverity": "WARNING", - "value": 0 - }, - { - "alertSeverity": "CRITICAL", - "value": 0.1 - } - ] - } - }, - { - "title": "Payload send results", - "layout": { - "column": 9, - "row": 1, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(count(logs.fb.payload.send.time), 1 minute) AS 'Status Code' FROM Metric FACET CASES(WHERE statusCode = 0 AS 'Send error') OR statusCode timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "REQUESTS_PER_MINUTE" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average timings", - "layout": { - "column": 5, - "row": 4, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT average(logs.fb.payload.send.time) AS 'Payload sending', average(logs.fb.packaging.time) AS 'Payload packaging' FROM Metric timeseries max" - } - ], - "nullValues": { - "nullValue": "zero" - }, - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "MS" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Accumulated time per minute", - "layout": { - "column": 9, - "row": 4, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.area" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(logs.fb.total.send.time), 1 minute) AS 'Sending', rate(sum(logs.fb.packaging.time), 1 minute) AS 'Packaging' FROM Metric TIMESERIES max" - } - ], - "nullValues": { - "nullValue": "zero" - }, - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "MS" - } - } - }, - { - "title": "Payload size", - "layout": { - "column": 5, - "row": 7, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT min(logs.fb.payload.size) AS 'Minimum', average(logs.fb.payload.size) AS 'Average', max(logs.fb.payload.size) AS 'Maximum' FROM Metric timeseries MAX " - } - ], - "nullValues": { - "nullValue": "default" - }, - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Payload packets per Fluent Bit chunk", - "layout": { - "column": 9, - "row": 7, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT min(logs.fb.payload.count) AS 'Minimum', average(logs.fb.payload.count) AS 'Average', max(logs.fb.payload.count) AS 'Maximum' FROM Metric timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "COUNT" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 7, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# INPUTS" + } + }, + { + "title": "Input byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" } - ] - } - ], - "variables": [ - { - "name": "cluster_name", - "items": null, - "defaultValues": [ - { - "value": { - "string": "*" - } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Input log rate (records/minute)", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" } - ], - "nrqlQuery": { - "accountId": 0, - "query": "SELECT uniques(cluster_name) FROM Metric where metricName = 'fluentbit_input_storage_overlimit'" + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average incoming record size (bytes)", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 }, - "options": { + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 11, + "width": 12, + "height": 1 }, - "title": "Cluster Name", - "type": "NRQL", - "isMultiSelection": true, - "replacementStrategy": "STRING" - } + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# FILTERS" + } + }, + { + "title": "Filter byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filter log rate (records/minute)", + "layout": { + "column": 5, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average filtered record size (bytes)", + "layout": { + "column": 9, + "row": 12, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Record add/drop rate per FILTER plugin", + "layout": { + "column": 1, + "row": 15, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "REQUESTS_PER_MINUTE" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 18, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# OUTPUTS" + } + }, + { + "title": "Output byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Output log rate (records/minute)", + "layout": { + "column": 5, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Average outgoing record size (bytes)", + "layout": { + "column": 9, + "row": 19, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin statistics (records/minute)", + "layout": { + "column": 1, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Other OUTPUT plugin statistics (records/minute)", + "layout": { + "column": 5, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'newrelic-logs-forwarder' and name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Connections per OUTPUT plugin", + "layout": { + "column": 9, + "row": 22, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_output_upstream_total_connections) as 'Total', max(fluentbit_output_upstream_busy_connections) as 'Busy' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin errors (errors/minute)", + "layout": { + "column": 1, + "row": 25, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "newrelic plugin chunk retry statistics (retries/minute)", + "layout": { + "column": 5, + "row": 25, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 28, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# MEMORY USAGE" + } + }, + { + "title": "Input plugin memory usage", + "layout": { + "column": 1, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "INPUT memory buffer over limit", + "layout": { + "column": 5, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#013ef4", + "seriesName": "pod-logs-tailer" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Chunk statistics per INPUT plugin", + "layout": { + "column": 9, + "row": 29, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Buffered chunks", + "layout": { + "column": 1, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_chunks) AS 'Total', max(fluentbit_storage_mem_chunks) AS 'Memory', max(fluentbit_storage_fs_chunks) AS 'Filesystem' FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Busy chunks' size", + "layout": { + "column": 5, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Filesystem chunks state", + "layout": { + "column": 9, + "row": 32, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT average(fluentbit_storage_fs_chunks_up) AS 'Up (in memory)', average(fluentbit_storage_fs_chunks_down) AS 'Down (fs only)' FROM Metric since '2024-02-29 13:22:00+0000' UNTIL '2024-02-29 14:31:00+0000' timeseries MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": [], + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } ] -} \ No newline at end of file + } \ No newline at end of file From 59deda8958a400d930e9ff0a7d5cb63c35ca027c Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 21:02:52 +0530 Subject: [PATCH 04/12] feat: push to check santize dashboard --- entity-types/ext-fluentbit/dashboard.json | 787 ++-------------------- 1 file changed, 67 insertions(+), 720 deletions(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index 335769007..42d764220 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -1,733 +1,151 @@ { - "name": "Kubernetes Fluent Bit monitoring", - "description": null, + "name": "Squid Cache Manager", "pages": [ { - "name": "Fluent Bit metrics: General", + "name": "Cache Manager", "description": null, "widgets": [ { - "title": "", + "title": "Active cache managers", "layout": { "column": 1, "row": 1, - "width": 6, - "height": 6 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" - } - }, - { - "title": "Fluent Bit version", - "layout": { - "column": 7, - "row": 1, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.table" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Fluent Bit uptime", - "layout": { - "column": 7, - "row": 4, - "width": 6, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 7, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# INPUTS" - } - }, - { - "title": "Input byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Input log rate (records/minute)", - "layout": { - "column": 5, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_input_records_total), 1 minute) as 'logs/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average incoming record size (bytes)", - "layout": { - "column": 9, - "row": 8, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_input_bytes_total)/sum(fluentbit_input_records_total) as 'Average incoming record size (bytes)' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 11, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# FILTERS" - } - }, - { - "title": "Filter byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_bytes_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Filter log rate (records/minute)", - "layout": { - "column": 5, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_records_total), 1 minute) FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average filtered record size (bytes)", - "layout": { - "column": 9, - "row": 12, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT sum(fluentbit_filter_bytes_total)/sum(fluentbit_filter_records_total) AS 'Average filtered record size (bytes)' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries max" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Record add/drop rate per FILTER plugin", - "layout": { - "column": 1, - "row": 15, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_filter_add_records_total), 1 minute) as 'Added back to pipeline', rate(sum(fluentbit_filter_drop_records_total), 1 minute) as 'Removed from pipeline' FROM Metric WHERE cluster_name IN ({{cluster_name}}) facet name, pod_name timeseries MAX" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "units": { - "unit": "REQUESTS_PER_MINUTE" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "", - "layout": { - "column": 1, - "row": 18, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# OUTPUTS" - } - }, - { - "title": "Output byte rate (bytes/minute)", - "layout": { - "column": 1, - "row": 19, - "width": 4, + "width": 3, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { "facet": { "showOtherSeries": false }, - "legend": { - "enabled": true - }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + "query": "FROM Metric SELECT latest(squid_up) compare with 1 hour ago" } ], "platformOptions": { "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "Output log rate (records/minute)", + "title": "Traffic ", "layout": { - "column": 5, - "row": 19, - "width": 4, + "column": 4, + "row": 1, + "width": 3, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ + "dataFormatters": [ { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'records/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX " + "name": "HTTP Traffic Out", + "type": "humanized" + }, + { + "name": "Total squid_client_http_kbytes_in_kbytes_total", + "type": "humanized" } ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Average outgoing record size (bytes)", - "layout": { - "column": 9, - "row": 19, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { "facet": { "showOtherSeries": false }, - "legend": { - "enabled": true - }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(fluentbit_output_proc_bytes_total)/sum(fluentbit_output_proc_records_total) as 'bytes' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + "query": "FROM Metric SELECT sum(squid_client_http_kbytes_in_kbytes_total) AS 'HTTP In (Kbs)', sum(squid_client_http_kbytes_out_kbytes_total) AS 'HTTP Out (Kbs)', sum(squid_client_http_requests_total) AS 'HTTP Requests' compare with 1 hour ago" } ], "platformOptions": { "ignoreTimeRange": false - }, - "thresholds": [], - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "newrelic plugin statistics (records/minute)", + "title": "Hit Rate (%)", "layout": { - "column": 1, - "row": 22, - "width": 4, + "column": 7, + "row": 1, + "width": 3, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ + "dataFormatters": [ { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + "name": "Avg squid_client_http_hit_kbytes_out_bytes_total", + "type": "humanized" } ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Other OUTPUT plugin statistics (records/minute)", - "layout": { - "column": 5, - "row": 22, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { "facet": { "showOtherSeries": false }, - "legend": { - "enabled": true - }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_proc_records_total), 1 minute) as 'Processed', rate(sum(fluentbit_output_dropped_records_total), 1 minute) as 'Dropped', rate(sum(fluentbit_output_retried_records_total), 1 minute) as 'Retried' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'newrelic-logs-forwarder' and name != 'fb-metrics-forwarder' facet name, pod_name timeseries max" + "query": "FROM Metric SELECT sum(squid_client_http_hit_kbytes_out_bytes_total) / sum(squid_client_http_kbytes_out_kbytes_total) * 100 As 'Bytes %', sum(squid_client_http_hits_total) / sum(squid_client_http_requests_total) * 100 AS 'Cache %'" } ], "platformOptions": { "ignoreTimeRange": false }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Connections per OUTPUT plugin", - "layout": { - "column": 9, - "row": 22, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ + "thresholds": [ { - "accountId": 0, - "query": "SELECT max(fluentbit_output_upstream_total_connections) as 'Total', max(fluentbit_output_upstream_busy_connections) as 'Busy' FROM Metric where cluster_name IN ({{cluster_name}}) AND name != 'fb-metrics-forwarder' facet name, pod_name timeseries MAX" + "alertSeverity": "CRITICAL" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } + ] } }, { - "title": "newrelic plugin errors (errors/minute)", + "title": "Error Rate (%)", "layout": { - "column": 1, - "row": 25, - "width": 4, + "column": 10, + "row": 1, + "width": 3, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ + "dataFormatters": [ { - "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_errors_total), 1 minute) AS 'Errors/minute' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries MAX " + "name": "Avg squid_client_http_errors_total", + "type": "decimal" } ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "newrelic plugin chunk retry statistics (retries/minute)", - "layout": { - "column": 5, - "row": 25, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { "facet": { "showOtherSeries": false }, - "legend": { - "enabled": true - }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(fluentbit_output_retries_total), 1 minute) as 'Retries', rate(sum(fluentbit_output_retries_failed_total), 1 minute) as 'Expirations' FROM Metric where cluster_name IN ({{cluster_name}}) AND name = 'newrelic-logs-forwarder' facet pod_name timeseries max" + "query": "FROM Metric SELECT sum(squid_client_http_errors_total) / sum(squid_client_http_requests_total) * 100 AS 'Error rate'" } ], "platformOptions": { "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "", - "layout": { - "column": 1, - "row": 28, - "width": 12, - "height": 1 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# MEMORY USAGE" - } - }, - { - "title": "Input plugin memory usage", + "title": "Traffic (Kbs)", "layout": { "column": 1, - "row": 29, + "row": 4, "width": 4, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.area" }, "rawConfiguration": { "facet": { @@ -739,45 +157,26 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_memory_bytes) as 'Max' FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name " + "query": "FROM Metric SELECT sum(squid_client_http_kbytes_in_kbytes_total) AS 'HTTP In (Kbs)', sum(squid_client_http_kbytes_out_kbytes_total) AS 'HTTP Out (Kbs)' timeseries auto" } ], "platformOptions": { "ignoreTimeRange": false - }, - "thresholds": [], - "units": { - "unit": "BYTES" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "INPUT memory buffer over limit", + "title": "Bytes Hit Rate (%)", "layout": { "column": 5, - "row": 29, + "row": 4, "width": 4, "height": 3 }, - "linkedEntityGuids": null, "visualization": { "id": "viz.line" }, "rawConfiguration": { - "colors": { - "seriesOverrides": [ - { - "color": "#013ef4", - "seriesName": "pod-logs-tailer" - } - ] - }, "facet": { "showOtherSeries": false }, @@ -787,32 +186,30 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_overlimit) FROM Metric where cluster_name IN ({{cluster_name}}) and name != 'fb-metrics-collector' timeseries max facet name, pod_name" + "query": "FROM Metric SELECT sum(squid_client_http_hit_kbytes_out_bytes_total) / sum(squid_client_http_kbytes_out_kbytes_total) TIMESERIES" } ], "platformOptions": { "ignoreTimeRange": false }, - "thresholds": [], - "yAxisLeft": { - "zero": true + "units": { + "unit": "PERCENTAGE" }, - "yAxisRight": { + "yAxisLeft": { "zero": true } } }, { - "title": "Chunk statistics per INPUT plugin", + "title": "Traffic requests", "layout": { "column": 9, - "row": 29, + "row": 4, "width": 4, "height": 3 }, - "linkedEntityGuids": null, "visualization": { - "id": "viz.line" + "id": "viz.area" }, "rawConfiguration": { "facet": { @@ -824,67 +221,22 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(fluentbit_input_storage_chunks_up) AS 'Up (in memory)', average(fluentbit_input_storage_chunks_down) AS 'Down (in fs)', average(fluentbit_input_storage_chunks_busy) AS 'Busy', average(fluentbit_input_storage_chunks) as 'Total' FROM Metric where name != 'fb-metrics-collector' since 1 hour ago timeseries MAX facet name, pod_name " + "query": "FROM Metric SELECT sum(squid_client_http_requests_total) AS 'HTTP Requests' timeseries auto" } ], "platformOptions": { "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "Buffered chunks", + "title": "Error Rate (%)", "layout": { "column": 1, - "row": 32, - "width": 4, - "height": 3 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_chunks) AS 'Total', max(fluentbit_storage_mem_chunks) AS 'Memory', max(fluentbit_storage_fs_chunks) AS 'Filesystem' FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries MAX " - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [], - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } - }, - { - "title": "Busy chunks' size", - "layout": { - "column": 5, - "row": 32, - "width": 4, + "row": 7, + "width": 8, "height": 3 }, - "linkedEntityGuids": null, "visualization": { "id": "viz.line" }, @@ -898,33 +250,28 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(fluentbit_input_storage_chunks_busy_bytes) FROM Metric where name != 'fb-metrics-collector' facet name, pod_name timeseries MAX since 1 hour ago" + "query": "FROM Metric SELECT sum(squid_client_http_errors_total) / sum(squid_client_http_requests_total) TIMESERIES" } ], "platformOptions": { "ignoreTimeRange": false }, - "thresholds": [], "units": { - "unit": "BYTES" + "unit": "PERCENTAGE" }, "yAxisLeft": { "zero": true - }, - "yAxisRight": { - "zero": true } } }, { - "title": "Filesystem chunks state", + "title": "Cache Hit Rate (%)", "layout": { "column": 9, - "row": 32, + "row": 7, "width": 4, "height": 3 }, - "linkedEntityGuids": null, "visualization": { "id": "viz.line" }, @@ -938,17 +285,16 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(fluentbit_storage_fs_chunks_up) AS 'Up (in memory)', average(fluentbit_storage_fs_chunks_down) AS 'Down (fs only)' FROM Metric since '2024-02-29 13:22:00+0000' UNTIL '2024-02-29 14:31:00+0000' timeseries MAX " + "query": "FROM Metric SELECT sum(squid_client_http_hits_total) / sum(squid_client_http_requests_total) TIMESERIES" } ], "platformOptions": { "ignoreTimeRange": false }, - "thresholds": [], - "yAxisLeft": { - "zero": true + "units": { + "unit": "PERCENTAGE" }, - "yAxisRight": { + "yAxisLeft": { "zero": true } } @@ -956,4 +302,5 @@ ] } ] - } \ No newline at end of file + } + \ No newline at end of file From cfb2a26ef787e18bfb069f9498b63d060052bf0c Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 21:48:54 +0530 Subject: [PATCH 05/12] feat: added widget 1 --- entity-types/ext-fluentbit/dashboard.json | 299 +--------------------- 1 file changed, 11 insertions(+), 288 deletions(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index 42d764220..20847f434 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -1,306 +1,29 @@ { - "name": "Squid Cache Manager", + "name": "Kubernetes Fluent Bit monitoring", + "description": null, "pages": [ { - "name": "Cache Manager", + "name": "Overview", "description": null, "widgets": [ { - "title": "Active cache managers", + "title": "", "layout": { "column": 1, "row": 1, - "width": 3, - "height": 3 + "width": 6, + "height": 6 }, + "linkedEntityGuids": null, "visualization": { - "id": "viz.billboard" + "id": "viz.markdown" }, "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT latest(squid_up) compare with 1 hour ago" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Traffic ", - "layout": { - "column": 4, - "row": 1, - "width": 3, - "height": 3 - }, - "visualization": { - "id": "viz.billboard" - }, - "rawConfiguration": { - "dataFormatters": [ - { - "name": "HTTP Traffic Out", - "type": "humanized" - }, - { - "name": "Total squid_client_http_kbytes_in_kbytes_total", - "type": "humanized" - } - ], - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_kbytes_in_kbytes_total) AS 'HTTP In (Kbs)', sum(squid_client_http_kbytes_out_kbytes_total) AS 'HTTP Out (Kbs)', sum(squid_client_http_requests_total) AS 'HTTP Requests' compare with 1 hour ago" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Hit Rate (%)", - "layout": { - "column": 7, - "row": 1, - "width": 3, - "height": 3 - }, - "visualization": { - "id": "viz.billboard" - }, - "rawConfiguration": { - "dataFormatters": [ - { - "name": "Avg squid_client_http_hit_kbytes_out_bytes_total", - "type": "humanized" - } - ], - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_hit_kbytes_out_bytes_total) / sum(squid_client_http_kbytes_out_kbytes_total) * 100 As 'Bytes %', sum(squid_client_http_hits_total) / sum(squid_client_http_requests_total) * 100 AS 'Cache %'" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": [ - { - "alertSeverity": "CRITICAL" - } - ] - } - }, - { - "title": "Error Rate (%)", - "layout": { - "column": 10, - "row": 1, - "width": 3, - "height": 3 - }, - "visualization": { - "id": "viz.billboard" - }, - "rawConfiguration": { - "dataFormatters": [ - { - "name": "Avg squid_client_http_errors_total", - "type": "decimal" - } - ], - "facet": { - "showOtherSeries": false - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_errors_total) / sum(squid_client_http_requests_total) * 100 AS 'Error rate'" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Traffic (Kbs)", - "layout": { - "column": 1, - "row": 4, - "width": 4, - "height": 3 - }, - "visualization": { - "id": "viz.area" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_kbytes_in_kbytes_total) AS 'HTTP In (Kbs)', sum(squid_client_http_kbytes_out_kbytes_total) AS 'HTTP Out (Kbs)' timeseries auto" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Bytes Hit Rate (%)", - "layout": { - "column": 5, - "row": 4, - "width": 4, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_hit_kbytes_out_bytes_total) / sum(squid_client_http_kbytes_out_kbytes_total) TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "PERCENTAGE" - }, - "yAxisLeft": { - "zero": true - } - } - }, - { - "title": "Traffic requests", - "layout": { - "column": 9, - "row": 4, - "width": 4, - "height": 3 - }, - "visualization": { - "id": "viz.area" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_requests_total) AS 'HTTP Requests' timeseries auto" - } - ], - "platformOptions": { - "ignoreTimeRange": false - } - } - }, - { - "title": "Error Rate (%)", - "layout": { - "column": 1, - "row": 7, - "width": 8, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_errors_total) / sum(squid_client_http_requests_total) TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "PERCENTAGE" - }, - "yAxisLeft": { - "zero": true - } - } - }, - { - "title": "Cache Hit Rate (%)", - "layout": { - "column": 9, - "row": 7, - "width": 4, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "FROM Metric SELECT sum(squid_client_http_hits_total) / sum(squid_client_http_requests_total) TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "PERCENTAGE" - }, - "yAxisLeft": { - "zero": true - } + "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" } } + ] } ] - } - \ No newline at end of file + } \ No newline at end of file From c1caed654d4e8e78f430106c31db5df25095d60b Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 21:53:13 +0530 Subject: [PATCH 06/12] feat: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 52 +++++++++++------------ 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index 20847f434..d42482da7 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -1,29 +1,27 @@ { - "name": "Kubernetes Fluent Bit monitoring", - "description": null, - "pages": [ - { - "name": "Overview", - "description": null, - "widgets": [ - { - "title": "", - "layout": { - "column": 1, - "row": 1, - "width": 6, - "height": 6 - }, - "linkedEntityGuids": null, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" - } + "name": "Kubernetes Fluent Bit monitoring", + "description": null, + "pages": [ + { + "name": "Overview", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 6 + }, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" } - - ] - } - ] - } \ No newline at end of file + } + ] + } + ] +} \ No newline at end of file From 866d644140346312fe0fbe3181d65532aa19c147 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:00:23 +0530 Subject: [PATCH 07/12] ci: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index d42482da7..2f34d131c 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -20,6 +20,32 @@ "rawConfiguration": { "text": "# README\n\n## About this page\nThis page represents most of [Fluent Bit's internal metrics](https://docs.fluentbit.io/manual/administration/monitoring#for-v2-metrics). The metric representations are grouped by categories and faceted by each plugin instance where appropriate.\n\n## How to filter\n1. Select the Kubernetes cluster you want to troubleshoot in the \"Cluster Name\" variable above.\n2. [OPTIONAL] You can use any of the values in the `Node name` and `Pod name` columns on the \"Fluent Bit version\" table to further filter the metrics displayed in the graphs below. To do so, you need to enable [facet filtering](https://docs.newrelic.com/docs/query-your-data/explore-query-data/dashboards/filter-new-relic-one-dashboards-facets/) on that table by clicking on the \"Edit\" submenu and select \"Filter the current dashboard\" under \"Facet Linking\". \n\n## Legend\n### Metric dimensions\n- **name**: the name of the Fluent Bit plugin. Version 1.21.0 of our Helm chart names them according to the plugin names described in the following section.\n- **pod_name**: the `newrelic-logging` pod (Fluent Bit instance) that emitted this metric.\n- **node_name**: physical Kubernetes node where the `newrelic-logging` pod is running.\n\n### Plugin names\n- **pod-logs-tailer**: `tail` *INPUT* plugin normally reading from `/var/log/containers/*.log`\n- **kubernetes-enricher**: `kubernetes` *FILTER* plugin that queries the Kubernetes API to enrich the logs with pod/container metadata.\n- **node-attributes-enricher**: `record_modifier` *FILTER* plugin that enriches logs with `cluster_name`.\n- **kubernetes-attribute-lifter** (only when in low data mode): `nest` *FILTER* plugin that lifts all the keys under `kubernetes`. This plugin is transparent to the final shape of the log.\n- **node-attributes-enricher-filter** (only when in low data mode): same as node-attributes-enricher`, but it also removes attributes that are not strictly necessary for correct platform functioning.\n- **newrelic-logs-forwarder**: `newrelic` *OUTPUT* plugin that sends logs to the New Relic Logs API" } + }, + { + "title": "Fluent Bit version", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": 0, + "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } } ] } From 2cd6bf3983ab069af97ed61aaddfffc7ed6bfd92 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:02:38 +0530 Subject: [PATCH 08/12] ci: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index 2f34d131c..4373a9e66 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -38,7 +38,7 @@ }, "nrqlQueries": [ { - "accountIds": 0, + "accountId": 0, "query": "SELECT latest(os) as 'OS', latest(version) as 'FB version', latest(cluster_name) FROM Metric where metricName = 'fluentbit_build_info' AND cluster_name IN ({{cluster_name}}) since 1 hour ago facet pod_name, node_name limit max" } ], From ef7d10edb92b17f36707eca62ae815d217521401 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:04:10 +0530 Subject: [PATCH 09/12] ci: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index 4373a9e66..aa968c320 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -46,6 +46,43 @@ "ignoreTimeRange": false } } + }, { + "title": "Fluent Bit uptime", + "layout": { + "column": 7, + "row": 4, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(fluentbit_uptime) FROM Metric where cluster_name IN ({{cluster_name}}) facet pod_name timeseries" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } } ] } From 4ffa6fee94f7b9534e8c000569723ac1c43a3956 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:07:14 +0530 Subject: [PATCH 10/12] ci: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index aa968c320..bfb919243 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -73,9 +73,6 @@ "platformOptions": { "ignoreTimeRange": false }, - "thresholds": { - "isLabelVisible": true - }, "yAxisLeft": { "zero": true }, From 6ce009476f2d1cc6a21da6b58a4490e1001cfd53 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:09:04 +0530 Subject: [PATCH 11/12] feat: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index bfb919243..dbbd795a5 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -80,6 +80,21 @@ "zero": true } } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 7, + "width": 12, + "height": 1 + }, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "# INPUTS" + } } ] } From 064118c6a74d026dda1a5a27695778191d867dc5 Mon Sep 17 00:00:00 2001 From: voorepreethi Date: Tue, 10 Dec 2024 22:10:24 +0530 Subject: [PATCH 12/12] ci: updated fb dashboard --- entity-types/ext-fluentbit/dashboard.json | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/entity-types/ext-fluentbit/dashboard.json b/entity-types/ext-fluentbit/dashboard.json index dbbd795a5..2747cadf3 100644 --- a/entity-types/ext-fluentbit/dashboard.json +++ b/entity-types/ext-fluentbit/dashboard.json @@ -95,6 +95,41 @@ "rawConfiguration": { "text": "# INPUTS" } + }, + { + "title": "Input byte rate (bytes/minute)", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(fluentbit_input_bytes_total), 1 minute) as 'bytes/minute' FROM Metric where name != 'fb-metrics-collector' and cluster_name IN ({{cluster_name}}) timeseries max facet name, pod_name" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } } ] }