diff --git a/example/docker-compose/grafana.yaml b/example/docker-compose/grafana.yaml index ee9b00cce61..94f092108f8 100644 --- a/example/docker-compose/grafana.yaml +++ b/example/docker-compose/grafana.yaml @@ -58,7 +58,7 @@ services: - MIMIR_ADDRESS=http://mimir:9009 - MIMIR_TENANT_ID=fake volumes: - - ../operations/alloy-mixin:/etc/alloy-mixin + - ../../operations/alloy-mixin:/etc/alloy-mixin working_dir: /etc/alloy-mixin command: grr watch . ./grizzly.jsonnet diff --git a/operations/alloy-mixin/config.libsonnet b/operations/alloy-mixin/config.libsonnet index 58fa6101c4f..152748b9de1 100644 --- a/operations/alloy-mixin/config.libsonnet +++ b/operations/alloy-mixin/config.libsonnet @@ -1,8 +1,8 @@ { _config+:: { - enableK8sCluster: true, - enableAlloyCluster: true, - enableLokiLogs: true, + enableK8sCluster: true, # Whether to add a cluster template variable to all dashboards + enableAlloyCluster: true, # Whether to add Alloy clustering dashboards + enableLokiLogs: true, filterSelector: '', #use it to filter specific metric label values, ie: job=~"integrations/alloy" k8sClusterSelector: 'cluster=~"$cluster", namespace=~"$namespace"', groupSelector: if self.enableK8sCluster then self.k8sClusterSelector + ', job=~"$job"' else 'job=~"$job"', diff --git a/operations/alloy-mixin/dashboards.libsonnet b/operations/alloy-mixin/dashboards.libsonnet index 6585154f27d..2a93ccfcfe0 100644 --- a/operations/alloy-mixin/dashboards.libsonnet +++ b/operations/alloy-mixin/dashboards.libsonnet @@ -8,6 +8,7 @@ (import './dashboards/controller.libsonnet') + (import './dashboards/prometheus.libsonnet') + (import './dashboards/opentelemetry.libsonnet') + + (import './dashboards/otel-engine-overview.libsonnet') + (import './dashboards/loki.libsonnet'), grafanaDashboards+:: diff --git a/operations/alloy-mixin/dashboards/otel-engine-overview.libsonnet b/operations/alloy-mixin/dashboards/otel-engine-overview.libsonnet new file mode 100644 index 00000000000..2e84fb0d67e --- /dev/null +++ b/operations/alloy-mixin/dashboards/otel-engine-overview.libsonnet @@ -0,0 +1,606 @@ +local dashboard = import './utils/dashboard.jsonnet'; +local panel = import './utils/panel.jsonnet'; +local filename = 'alloy-otel-engine-overview.json'; + +{ + local templateVariables = [ + dashboard.newMultiTemplateVariable( + 'cluster', + 'label_values(otelcol_process_uptime_seconds_total, cluster)', + setenceCaseLabels=$._config.useSetenceCaseTemplateLabels, + ), + dashboard.newMultiTemplateVariable( + 'namespace', + 'label_values(otelcol_process_uptime_seconds_total{cluster=~"$cluster"}, namespace)', + setenceCaseLabels=$._config.useSetenceCaseTemplateLabels, + ), + dashboard.newMultiTemplateVariable( + 'job', + 'label_values(otelcol_process_uptime_seconds_total{cluster=~"$cluster", namespace=~"$namespace"}, job)', + setenceCaseLabels=$._config.useSetenceCaseTemplateLabels, + ), + dashboard.newTemplateVariableCustom('groupby', 'instance,receiver,transport,exporter,processor,otel_signal,otel_scope_name,job,namespace,cluster,pod') { label: 'Group by' }, + { + name: 'filters', + type: 'adhoc', + datasource: { + type: 'prometheus', + uid: '${datasource}', + }, + }, + ], + + local panelPosition3Across(row, col) = panel.withPosition({ x: col * 8, y: row * 10, w: 8, h: 10 }), + local rowPosition(row) = panel.withPosition({ h: 1, w: 24, x: 0, y: row * 10 }), + + local signalRow( + title, + rowNum, + signalName, + receiverAccepted, + receiverRefused, + exporterSent, + exporterSendFailed, + exporterEnqueueFailed, + queueDataType, + ) = ( + panel.new(title, 'row') + + rowPosition(rowNum) + + { collapsed: true } + + { + local rateQuery(metric) = ('rate(%s{%%(groupSelector)s}[$__rate_interval])' % metric) % $._config, + + panels: [ + ( + panel.new(title='Receiver: Total accepted & refused', type='timeseries') + + panel.withDescription( + 'Total rate of accepted and refused %s across all receivers.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum, col=0) + + panel.withQueries([ + panel.newQuery( + expr='sum(%s)' % rateQuery(receiverAccepted), + legendFormat='accepted', + ), + panel.newQuery( + expr='sum(%s)' % rateQuery(receiverRefused), + legendFormat='refused', + ), + ]) + ), + ( + panel.new(title='Receiver: Accepted by ${groupby}', type='timeseries') + + panel.withDescription( + 'Accepted and refused %s rates broken down by the selected dimension.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum, col=1) + + panel.withQueries([ + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(receiverAccepted), + legendFormat='{{${groupby}}} accepted', + ), + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(receiverRefused), + legendFormat='{{${groupby}}} refused', + ), + ]) + ), + ( + panel.new(title='Receiver: Refused by ${groupby}', type='timeseries') + + panel.withDescription( + 'Refused %s rate broken down by the selected dimension.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum, col=2) + + panel.withQueries([ + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(receiverRefused), + legendFormat='{{${groupby}}}', + ), + ]) + ), + ( + panel.new(title='Exporter: Total sent & failed', type='timeseries') + + panel.withDescription( + 'Total rate of sent, send-failed, and enqueue-failed %s across all exporters.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum + 1, col=0) + + panel.withQueries([ + panel.newQuery( + expr='sum(%s)' % rateQuery(exporterSent), + legendFormat='sent', + ), + panel.newQuery( + expr='sum(%s)' % rateQuery(exporterSendFailed), + legendFormat='send failed', + ), + panel.newQuery( + expr='sum(%s)' % rateQuery(exporterEnqueueFailed), + legendFormat='enqueue failed', + ), + ]) + ), + ( + panel.new(title='Exporter: Sent by ${groupby}', type='timeseries') + + panel.withDescription( + 'Sent %s rate broken down by the selected dimension.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum + 1, col=1) + + panel.withQueries([ + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(exporterSent), + legendFormat='{{${groupby}}} sent', + ), + ]) + ), + ( + panel.new(title='Exporter: Failed by ${groupby}', type='timeseries') + + panel.withDescription( + 'Send-failed and enqueue-failed %s rates broken down by the selected dimension.' % signalName, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=rowNum + 1, col=2) + + panel.withQueries([ + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(exporterSendFailed), + legendFormat='{{${groupby}}} send failed', + ), + panel.newQuery( + expr='sum by(${groupby}) (%s)' % rateQuery(exporterEnqueueFailed), + legendFormat='{{${groupby}}} enqueue failed', + ), + ]) + ), + ( + panel.new(title='Exporter: Queue utilization by ${groupby}', type='timeseries') + + panel.withDescription( + 'Exporter send queue usage as a fraction of capacity. Values approaching 1 indicate backpressure and risk of data loss.', + ) + + panel.withUnit('percentunit') + + panelPosition3Across(row=rowNum + 2, col=0) + + panel.withQueries([ + panel.newQuery( + expr=(||| + sum by(${groupby}) (otelcol_exporter_queue_size{%%(groupSelector)s, data_type="%s"}) + / + clamp_min(sum by(${groupby}) (otelcol_exporter_queue_capacity{%%(groupSelector)s, data_type="%s"}), 1) + ||| % [queueDataType, queueDataType]) % $._config, + legendFormat='{{${groupby}}}', + ), + ]) + ), + ], + } + ), + + [filename]: + dashboard.new(name='Alloy / OTel Engine Overview', tag=$._config.dashboardTag) + + { description: 'Overview of the OpenTelemetry (OTel) engine running inside Alloy.' } + + dashboard.withDocsLink( + url='https://grafana.com/docs/alloy/latest/opentelemetry/', + desc='OTel Engine documentation', + ) + + dashboard.withDashboardsLink(tag=$._config.dashboardTag) + + dashboard.withUID(std.md5(filename)) + + dashboard.withTemplateVariablesMixin(templateVariables) + + dashboard.withPanelsMixin([ + // Overview row + ( + panel.new('Overview', 'row') + + rowPosition(0) + ), + ( + panel.new(title='Instances Count', type='timeseries') + + panel.withDescription(||| + Number of instances with OTel engine metrics. + |||) + + panelPosition3Across(row=0, col=0) + + panel.withQueries([ + panel.newQuery( + expr=||| + count(otelcol_process_uptime_seconds_total{%(groupSelector)s}) + ||| % $._config, + legendFormat='count', + ), + ]) + ), + ( + panel.new(title='Receivers SR', type='timeseries') + + panel.withDescription(||| + Receiver success rate for each signal type. + |||) + + panel.withUnit('percentunit') + + panelPosition3Across(row=0, col=1) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum(rate(otelcol_receiver_accepted_spans_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_receiver_accepted_spans_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_receiver_refused_spans_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='spans', + ), + panel.newQuery( + expr=||| + sum(rate(otelcol_receiver_accepted_metric_points_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_receiver_accepted_metric_points_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_receiver_refused_metric_points_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='metric points', + ), + panel.newQuery( + expr=||| + sum(rate(otelcol_receiver_accepted_log_records_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_receiver_accepted_log_records_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_receiver_refused_log_records_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='log records', + ), + ]) + ), + ( + panel.new(title='Exporters SR', type='timeseries') + + panel.withDescription(||| + Exporter success rate for each signal type. + |||) + + panel.withUnit('percentunit') + + panelPosition3Across(row=0, col=2) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum(rate(otelcol_exporter_sent_spans_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_exporter_sent_spans_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_exporter_enqueue_failed_spans_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + + + (sum(rate(otelcol_exporter_send_failed_spans_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='spans', + ), + panel.newQuery( + expr=||| + sum(rate(otelcol_exporter_sent_metric_points_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_exporter_sent_metric_points_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_exporter_enqueue_failed_metric_points_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + + + (sum(rate(otelcol_exporter_send_failed_metric_points_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='metric points', + ), + panel.newQuery( + expr=||| + sum(rate(otelcol_exporter_sent_log_records_total{%(groupSelector)s}[$__rate_interval])) + / + ( + sum(rate(otelcol_exporter_sent_log_records_total{%(groupSelector)s}[$__rate_interval])) + + + (sum(rate(otelcol_exporter_enqueue_failed_log_records_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + + + (sum(rate(otelcol_exporter_send_failed_log_records_total{%(groupSelector)s}[$__rate_interval])) or vector(0)) + ) + ||| % $._config, + legendFormat='log records', + ), + ]) + ), + + // Signal-specific rows (collapsed by default) + signalRow( + title='Spans & Traces', + rowNum=2, + signalName='spans', + receiverAccepted='otelcol_receiver_accepted_spans_total', + receiverRefused='otelcol_receiver_refused_spans_total', + exporterSent='otelcol_exporter_sent_spans_total', + exporterSendFailed='otelcol_exporter_send_failed_spans_total', + exporterEnqueueFailed='otelcol_exporter_enqueue_failed_spans_total', + queueDataType='traces', + ), + signalRow( + title='Metrics', + rowNum=5, + signalName='metric points', + receiverAccepted='otelcol_receiver_accepted_metric_points_total', + receiverRefused='otelcol_receiver_refused_metric_points_total', + exporterSent='otelcol_exporter_sent_metric_points_total', + exporterSendFailed='otelcol_exporter_send_failed_metric_points_total', + exporterEnqueueFailed='otelcol_exporter_enqueue_failed_metric_points_total', + queueDataType='metrics', + ), + signalRow( + title='Logs', + rowNum=8, + signalName='log records', + receiverAccepted='otelcol_receiver_accepted_log_records_total', + receiverRefused='otelcol_receiver_refused_log_records_total', + exporterSent='otelcol_exporter_sent_log_records_total', + exporterSendFailed='otelcol_exporter_send_failed_log_records_total', + exporterEnqueueFailed='otelcol_exporter_enqueue_failed_log_records_total', + queueDataType='logs', + ), + + // Processing & Batching row (collapsed by default) + ( + panel.new('Processing & Batching', 'row') + + rowPosition(11) + + { collapsed: true } + + { + local normalNote = 'Requires "normal" telemetry level.', + + panels: [ + // Row 1: Processor throughput + ( + panel.new(title='Processor throughput', type='timeseries') + + panel.withDescription( + 'Incoming vs outgoing items across all processors. A gap between the two may indicate items being dropped or filtered.', + ) + + panel.withUnit('cps') + + panelPosition3Across(row=11, col=0) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum(rate(otelcol_processor_incoming_items_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='incoming', + ), + panel.newQuery( + expr=||| + sum(rate(otelcol_processor_outgoing_items_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='outgoing', + ), + ]) + ), + ( + panel.new(title='Processor throughput by ${groupby}', type='timeseries') + + panel.withDescription( + 'Incoming vs outgoing items broken down by the selected dimension.', + ) + + panel.withUnit('cps') + + panelPosition3Across(row=11, col=1) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_incoming_items_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} incoming', + ), + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_outgoing_items_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} outgoing', + ), + ]) + ), + ( + panel.new(title='Processor refused by ${groupby}', type='timeseries') + + panel.withDescription( + 'Items refused by processors, broken down by signal type and selected dimension.', + ) + + panel.withUnit('cps') + + panelPosition3Across(row=11, col=2) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_refused_spans_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} spans', + ), + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_refused_metric_points_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} metric points', + ), + ]) + ), + // Row 2: Batching + ( + panel.newHeatmap('Batch send size', 'short') + + panel.withDescription( + 'Distribution of batch sizes when sent. Shows how full batches are before being flushed. ' + normalNote, + ) + + panelPosition3Across(row=12, col=0) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by (le) (increase(otelcol_processor_batch_batch_send_size_bucket{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + format='heatmap', + legendFormat='{{le}}', + ), + ]) + ), + ( + panel.new(title='Batch send triggers by ${groupby}', type='timeseries') + + panel.withDescription( + 'How batches are flushed: by reaching the size limit or by timeout. Mostly timeout triggers may indicate low throughput or a large batch size setting. ' + normalNote, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=12, col=1) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_batch_batch_size_trigger_send_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} size trigger', + ), + panel.newQuery( + expr=||| + sum by(${groupby}) (rate(otelcol_processor_batch_timeout_trigger_send_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='{{${groupby}}} timeout trigger', + ), + ]) + ), + ( + panel.new(title='Batch metadata cardinality by ${groupby}', type='timeseries') + + panel.withDescription( + 'Number of distinct metadata value combinations being processed. High cardinality increases memory usage and may hit the metadata_cardinality_limit. ' + normalNote, + ) + + panelPosition3Across(row=12, col=2) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by(${groupby}) (otelcol_processor_batch_metadata_cardinality{%(groupSelector)s}) + ||| % $._config, + legendFormat='{{${groupby}}}', + ), + ]) + ), + ], + } + ), + + // Process Resources row (collapsed by default) + ( + panel.new('Process Resources', 'row') + + rowPosition(13) + + { collapsed: true } + + { + local avgLineStyle = [ + { id: 'color', value: { fixedColor: 'green', mode: 'fixed' } }, + { id: 'custom.lineStyle', value: { dash: [10, 10], fill: 'dash' } }, + { id: 'custom.lineWidth', value: 2 }, + ], + + panels: [ + ( + panel.new(title='CPU usage', type='timeseries') + + panel.withDescription( + 'CPU cores used by OTel engine processes. 1.0 = 100% of one core.', + ) + + panel.withUnit('short') + + panelPosition3Across(row=13, col=0) + + panel.withQueries([ + panel.newQuery( + expr=||| + avg(sum by(instance) (rate(otelcol_process_cpu_seconds_total{%(groupSelector)s}[$__rate_interval]))) + ||| % $._config, + legendFormat='average', + ), + panel.newQuery( + expr=||| + sum by(instance) (rate(otelcol_process_cpu_seconds_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + ), + ]) + + panel.withOverridesByName('average', avgLineStyle) + ), + ( + panel.new(title='Memory RSS', type='timeseries') + + panel.withDescription( + 'Resident set size (physical memory) used by OTel engine processes.', + ) + + panel.withUnit('bytes') + + panelPosition3Across(row=13, col=1) + + panel.withQueries([ + panel.newQuery( + expr=||| + avg(sum by(instance) (otelcol_process_memory_rss_bytes{%(groupSelector)s})) + ||| % $._config, + legendFormat='average', + ), + panel.newQuery( + expr=||| + sum by(instance) (otelcol_process_memory_rss_bytes{%(groupSelector)s}) + ||| % $._config, + ), + ]) + + panel.withOverridesByName('average', avgLineStyle) + ), + ( + panel.new(title='Go allocation rate', type='timeseries') + + panel.withDescription( + 'Rate of heap memory allocation. High allocation rates often lead to GC pressure.', + ) + + panel.withUnit('Bps') + + panelPosition3Across(row=13, col=2) + + panel.withQueries([ + panel.newQuery( + expr=||| + avg(sum by(instance) (rate(otelcol_process_runtime_total_alloc_bytes_total{%(groupSelector)s}[$__rate_interval]))) + ||| % $._config, + legendFormat='average', + ), + panel.newQuery( + expr=||| + sum by(instance) (rate(otelcol_process_runtime_total_alloc_bytes_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + ), + ]) + + panel.withOverridesByName('average', avgLineStyle) + ), + ], + } + ), + + // Grafana Cloud connector row (collapsed by default) + ( + panel.new('Grafana Cloud Connector', 'row') + + rowPosition(15) + + { collapsed: true } + + { + local connectorNote = 'Only available when the grafanacloud connector is configured.', + + panels: [ + ( + panel.new(title='Unique hosts tracked', type='timeseries') + + panel.withDescription( + 'Number of unique hosts identified by the grafanacloud connector for Application Observability billing. ' + connectorNote, + ) + + panelPosition3Across(row=15, col=0) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum by(${groupby}) (otelcol_grafanacloud_host_count_ratio{%(groupSelector)s}) + ||| % $._config, + legendFormat='{{${groupby}}}', + ), + ]) + ), + ( + panel.new(title='Datapoints sent rate', type='timeseries') + + panel.withDescription( + 'Total rate of datapoints sent to Grafana Cloud by the grafanacloud connector. ' + connectorNote, + ) + + panel.withUnit('cps') + + panelPosition3Across(row=15, col=1) + + panel.withQueries([ + panel.newQuery( + expr=||| + sum(rate(otelcol_grafanacloud_datapoint_count_total{%(groupSelector)s}[$__rate_interval])) + ||| % $._config, + legendFormat='total', + ), + ]) + ), + ], + } + ), + ]), +} diff --git a/operations/alloy-mixin/dashboards/utils/dashboard.jsonnet b/operations/alloy-mixin/dashboards/utils/dashboard.jsonnet index 770ad556e64..846061c7d0d 100644 --- a/operations/alloy-mixin/dashboards/utils/dashboard.jsonnet +++ b/operations/alloy-mixin/dashboards/utils/dashboard.jsonnet @@ -76,7 +76,7 @@ }, datasource: '${datasource}', refresh: 2, - sort: 2, + sort: 1, }, newTemplateVariableCustom(name, query) :: { diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-cluster-node.json b/operations/alloy-mixin/rendered/dashboards/alloy-cluster-node.json index 71a974bb4a2..32e49434b55 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-cluster-node.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-cluster-node.json @@ -425,7 +425,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -437,7 +437,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -449,7 +449,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -464,7 +464,7 @@ "refId": "instance" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-cluster-overview.json b/operations/alloy-mixin/rendered/dashboards/alloy-cluster-overview.json index 33d82693080..6e68ef4dc78 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-cluster-overview.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-cluster-overview.json @@ -356,7 +356,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -368,7 +368,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -380,7 +380,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-controller.json b/operations/alloy-mixin/rendered/dashboards/alloy-controller.json index fd28fcbbfd2..21b2a26f3c8 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-controller.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-controller.json @@ -489,7 +489,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -501,7 +501,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -513,7 +513,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-loki.json b/operations/alloy-mixin/rendered/dashboards/alloy-loki.json index b74134fde85..3b06eec6ac8 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-loki.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-loki.json @@ -329,7 +329,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -341,7 +341,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -353,7 +353,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -368,7 +368,7 @@ "refId": "instance" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -383,7 +383,7 @@ "refId": "url" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-opentelemetry.json b/operations/alloy-mixin/rendered/dashboards/alloy-opentelemetry.json index 31aff5fd2fe..37d75cd46fd 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-opentelemetry.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-opentelemetry.json @@ -774,7 +774,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -786,7 +786,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -798,7 +798,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -813,7 +813,7 @@ "refId": "instance" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-otel-engine-overview.json b/operations/alloy-mixin/rendered/dashboards/alloy-otel-engine-overview.json new file mode 100644 index 00000000000..492fcdf0bf4 --- /dev/null +++ b/operations/alloy-mixin/rendered/dashboards/alloy-otel-engine-overview.json @@ -0,0 +1,1432 @@ +{ + "description": "Overview of the OpenTelemetry (OTel) engine running inside Alloy.", + "graphTooltip": 1, + "links": [ + { + "icon": "doc", + "targetBlank": true, + "title": "Documentation", + "tooltip": "OTel Engine documentation", + "type": "link", + "url": "https://grafana.com/docs/alloy/latest/opentelemetry/" + }, + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "alloy-mixin" + ], + "targetBlank": false, + "title": "Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "title": "Overview", + "type": "row" + }, + { + "datasource": "${datasource}", + "description": "Number of instances with OTel engine metrics.\n", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 0 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "count(otelcol_process_uptime_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"})\n", + "instant": false, + "legendFormat": "count", + "range": true + } + ], + "title": "Instances Count", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Receiver success rate for each signal type.\n", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 0 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_receiver_accepted_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_receiver_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "spans", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_receiver_accepted_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_receiver_refused_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "metric points", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_receiver_accepted_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_receiver_refused_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "log records", + "range": true + } + ], + "title": "Receivers SR", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Exporter success rate for each signal type.\n", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 0 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_exporter_sent_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_exporter_enqueue_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n +\n (sum(rate(otelcol_exporter_send_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "spans", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_exporter_sent_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_exporter_enqueue_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n +\n (sum(rate(otelcol_exporter_send_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "metric points", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n/\n(\n sum(rate(otelcol_exporter_sent_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n +\n (sum(rate(otelcol_exporter_enqueue_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n +\n (sum(rate(otelcol_exporter_send_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])) or vector(0))\n)\n", + "instant": false, + "legendFormat": "log records", + "range": true + } + ], + "title": "Exporters SR", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "Total rate of accepted and refused spans across all receivers.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 20 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "refused", + "range": true + } + ], + "title": "Receiver: Total accepted & refused", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Accepted and refused spans rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 20 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_accepted_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} refused", + "range": true + } + ], + "title": "Receiver: Accepted by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Refused spans rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 20 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Receiver: Refused by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Total rate of sent, send-failed, and enqueue-failed spans across all exporters.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 30 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "sent", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_enqueue_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "enqueue failed", + "range": true + } + ], + "title": "Exporter: Total sent & failed", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Sent spans rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 30 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_sent_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} sent", + "range": true + } + ], + "title": "Exporter: Sent by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Send-failed and enqueue-failed spans rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 30 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_send_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_enqueue_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} enqueue failed", + "range": true + } + ], + "title": "Exporter: Failed by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Exporter send queue usage as a fraction of capacity. Values approaching 1 indicate backpressure and risk of data loss.", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 40 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (otelcol_exporter_queue_size{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"traces\"})\n/\nclamp_min(sum by(${groupby}) (otelcol_exporter_queue_capacity{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"traces\"}), 1)\n", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Exporter: Queue utilization by ${groupby}", + "type": "timeseries" + } + ], + "title": "Spans & Traces", + "type": "row" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "Total rate of accepted and refused metric points across all receivers.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 50 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_refused_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "refused", + "range": true + } + ], + "title": "Receiver: Total accepted & refused", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Accepted and refused metric points rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 50 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_accepted_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} refused", + "range": true + } + ], + "title": "Receiver: Accepted by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Refused metric points rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 50 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Receiver: Refused by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Total rate of sent, send-failed, and enqueue-failed metric points across all exporters.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 60 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "sent", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_enqueue_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "enqueue failed", + "range": true + } + ], + "title": "Exporter: Total sent & failed", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Sent metric points rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 60 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_sent_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} sent", + "range": true + } + ], + "title": "Exporter: Sent by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Send-failed and enqueue-failed metric points rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 60 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_send_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_enqueue_failed_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} enqueue failed", + "range": true + } + ], + "title": "Exporter: Failed by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Exporter send queue usage as a fraction of capacity. Values approaching 1 indicate backpressure and risk of data loss.", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 70 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (otelcol_exporter_queue_size{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"metrics\"})\n/\nclamp_min(sum by(${groupby}) (otelcol_exporter_queue_capacity{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"metrics\"}), 1)\n", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Exporter: Queue utilization by ${groupby}", + "type": "timeseries" + } + ], + "title": "Metrics", + "type": "row" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 80 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "Total rate of accepted and refused log records across all receivers.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 80 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_accepted_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_receiver_refused_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "refused", + "range": true + } + ], + "title": "Receiver: Total accepted & refused", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Accepted and refused log records rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 80 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_accepted_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} accepted", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} refused", + "range": true + } + ], + "title": "Receiver: Accepted by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Refused log records rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 80 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_receiver_refused_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Receiver: Refused by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Total rate of sent, send-failed, and enqueue-failed log records across all exporters.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 90 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_sent_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "sent", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_exporter_enqueue_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "enqueue failed", + "range": true + } + ], + "title": "Exporter: Total sent & failed", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Sent log records rate broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 90 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_sent_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} sent", + "range": true + } + ], + "title": "Exporter: Sent by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Send-failed and enqueue-failed log records rates broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 90 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_send_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} send failed", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_exporter_enqueue_failed_log_records_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{${groupby}}} enqueue failed", + "range": true + } + ], + "title": "Exporter: Failed by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Exporter send queue usage as a fraction of capacity. Values approaching 1 indicate backpressure and risk of data loss.", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 100 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (otelcol_exporter_queue_size{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"logs\"})\n/\nclamp_min(sum by(${groupby}) (otelcol_exporter_queue_capacity{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", data_type=\"logs\"}), 1)\n", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Exporter: Queue utilization by ${groupby}", + "type": "timeseries" + } + ], + "title": "Logs", + "type": "row" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 110 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "Incoming vs outgoing items across all processors. A gap between the two may indicate items being dropped or filtered.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 110 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_processor_incoming_items_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "incoming", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_processor_outgoing_items_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "outgoing", + "range": true + } + ], + "title": "Processor throughput", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Incoming vs outgoing items broken down by the selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 110 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_incoming_items_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} incoming", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_outgoing_items_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} outgoing", + "range": true + } + ], + "title": "Processor throughput by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Items refused by processors, broken down by signal type and selected dimension.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 110 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} spans", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_refused_metric_points_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} metric points", + "range": true + } + ], + "title": "Processor refused by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Distribution of batch sizes when sent. Shows how full batches are before being flushed. Requires \"normal\" telemetry level.", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 120 + }, + "maxDataPoints": 30, + "options": { + "calculate": false, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "scale": "exponential", + "scheme": "Oranges", + "steps": 65 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "unit": "short" + } + }, + "pluginVersion": "9.0.6", + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by (le) (increase(otelcol_processor_batch_batch_send_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "format": "heatmap", + "instant": false, + "legendFormat": "{{le}}", + "range": true + } + ], + "title": "Batch send size", + "type": "heatmap" + }, + { + "datasource": "${datasource}", + "description": "How batches are flushed: by reaching the size limit or by timeout. Mostly timeout triggers may indicate low throughput or a large batch size setting. Requires \"normal\" telemetry level.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 120 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_batch_batch_size_trigger_send_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} size trigger", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (rate(otelcol_processor_batch_timeout_trigger_send_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "{{${groupby}}} timeout trigger", + "range": true + } + ], + "title": "Batch send triggers by ${groupby}", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Number of distinct metadata value combinations being processed. High cardinality increases memory usage and may hit the metadata_cardinality_limit. Requires \"normal\" telemetry level.", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 120 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (otelcol_processor_batch_metadata_cardinality{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"})\n", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Batch metadata cardinality by ${groupby}", + "type": "timeseries" + } + ], + "title": "Processing & Batching", + "type": "row" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 130 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "CPU cores used by OTel engine processes. 1.0 = 100% of one core.", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "average" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 130 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "avg(sum by(instance) (rate(otelcol_process_cpu_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])))\n", + "instant": false, + "legendFormat": "average", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(instance) (rate(otelcol_process_cpu_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "__auto", + "range": true + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Resident set size (physical memory) used by OTel engine processes.", + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "average" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 130 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "avg(sum by(instance) (otelcol_process_memory_rss_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}))\n", + "instant": false, + "legendFormat": "average", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(instance) (otelcol_process_memory_rss_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"})\n", + "instant": false, + "legendFormat": "__auto", + "range": true + } + ], + "title": "Memory RSS", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Rate of heap memory allocation. High allocation rates often lead to GC pressure.", + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "average" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 130 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "avg(sum by(instance) (rate(otelcol_process_runtime_total_alloc_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval])))\n", + "instant": false, + "legendFormat": "average", + "range": true + }, + { + "datasource": "${datasource}", + "expr": "sum by(instance) (rate(otelcol_process_runtime_total_alloc_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "__auto", + "range": true + } + ], + "title": "Go allocation rate", + "type": "timeseries" + } + ], + "title": "Process Resources", + "type": "row" + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 150 + }, + "panels": [ + { + "datasource": "${datasource}", + "description": "Number of unique hosts identified by the grafanacloud connector for Application Observability billing. Only available when the grafanacloud connector is configured.", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 150 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum by(${groupby}) (otelcol_grafanacloud_host_count_ratio{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"})\n", + "instant": false, + "legendFormat": "{{${groupby}}}", + "range": true + } + ], + "title": "Unique hosts tracked", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Total rate of datapoints sent to Grafana Cloud by the grafanacloud connector. Only available when the grafanacloud connector is configured.", + "fieldConfig": { + "defaults": { + "unit": "cps" + } + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 150 + }, + "targets": [ + { + "datasource": "${datasource}", + "expr": "sum(rate(otelcol_grafanacloud_datapoint_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}[$__rate_interval]))\n", + "instant": false, + "legendFormat": "total", + "range": true + } + ], + "title": "Datapoints sent rate", + "type": "timeseries" + } + ], + "title": "Grafana Cloud Connector", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 36, + "tags": [ + "alloy-mixin" + ], + "templating": { + "list": [ + { + "label": "Data Source", + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "sort": 2, + "type": "datasource" + }, + { + "label": "Loki Data Source", + "name": "loki_datasource", + "query": "loki", + "refresh": 1, + "sort": 2, + "type": "datasource" + }, + { + "allValue": ".*", + "datasource": "${datasource}", + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "query": { + "query": "label_values(otelcol_process_uptime_seconds_total, cluster)", + "refId": "cluster" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "datasource": "${datasource}", + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "query": { + "query": "label_values(otelcol_process_uptime_seconds_total{cluster=~\"$cluster\"}, namespace)", + "refId": "namespace" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "datasource": "${datasource}", + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "query": { + "query": "label_values(otelcol_process_uptime_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}, job)", + "refId": "job" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "label": "Group by", + "name": "groupby", + "query": "instance,receiver,transport,exporter,processor,otel_signal,otel_scope_name,job,namespace,cluster,pod", + "type": "custom" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "name": "filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d", + "90d" + ] + }, + "timezone": "utc", + "title": "Alloy / OTel Engine Overview", + "uid": "975a324565222be2906d572d6fc960ff" +} diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-prometheus-remote-write.json b/operations/alloy-mixin/rendered/dashboards/alloy-prometheus-remote-write.json index 8838508489d..754ac4d2d8a 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-prometheus-remote-write.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-prometheus-remote-write.json @@ -568,7 +568,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -580,7 +580,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -592,7 +592,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -607,7 +607,7 @@ "refId": "instance" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -622,7 +622,7 @@ "refId": "component_path" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -637,7 +637,7 @@ "refId": "component" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -652,7 +652,7 @@ "refId": "url" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ] diff --git a/operations/alloy-mixin/rendered/dashboards/alloy-resources.json b/operations/alloy-mixin/rendered/dashboards/alloy-resources.json index 05e48cec2b5..edc06451e89 100644 --- a/operations/alloy-mixin/rendered/dashboards/alloy-resources.json +++ b/operations/alloy-mixin/rendered/dashboards/alloy-resources.json @@ -261,7 +261,7 @@ "refId": "cluster" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -273,7 +273,7 @@ "refId": "namespace" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -285,7 +285,7 @@ "refId": "job" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" }, { @@ -300,7 +300,7 @@ "refId": "instance" }, "refresh": 2, - "sort": 2, + "sort": 1, "type": "query" } ]