diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 06fbbe96..dbbb1a4c 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -273,6 +273,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), depends_on: DependsOnOptions::Simple(vec!["cb_prometheus".to_owned()]), environment: Environment::List(vec!["GF_SECURITY_ADMIN_PASSWORD=admin".to_owned()]), + volumes: vec![Volumes::Simple("./grafana/dashboards:/etc/grafana/provisioning/dashboards".to_owned()), Volumes::Simple("./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned())], // TODO: re-enable logging here once we move away from docker logs logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), ..Service::default() diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index d7f21dd7..917e8047 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -16,7 +16,7 @@ lazy_static! { // FROM RELAYS /// Status code received by relay by endpoint pub static ref RELAY_STATUS_CODE: IntCounterVec = register_int_counter_vec_with_registry!( - "relay_status_code", + "relay_status_code_total", "HTTP status code received by relay", &["http_status_code", "endpoint", "relay_id"], PBS_METRICS_REGISTRY @@ -35,7 +35,7 @@ lazy_static! { // TO BEACON NODE /// Status code returned to beacon node by endpoint pub static ref BEACON_NODE_STATUS: IntCounterVec = register_int_counter_vec_with_registry!( - "beacon_node_status_code", + "beacon_node_status_code_total", "HTTP status code returned to beacon node", &["http_status_code", "endpoint"], PBS_METRICS_REGISTRY diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json new file mode 100644 index 00000000..591e72fb --- /dev/null +++ b/grafana/dashboards/dashboard.json @@ -0,0 +1,803 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "repeat": "endpoint", + "repeatDirection": "h", + "title": "$endpoint calls", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_relay_status_code_total{http_status_code=~\"2..\", endpoint=\"$endpoint\"}[1h])) by (relay_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Relay Success QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_relay_status_code_total{http_status_code=~\"4..|5..\", endpoint=\"$endpoint\"}[1h])) by (relay_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Relay Error QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_beacon_node_status_code_total{http_status_code=~\"2..\", endpoint=\"$endpoint\"}[1h]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Beacon Node Success QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "editorMode": "code", + "expr": "sum(increase(cb_pbs_beacon_node_status_code_total{http_status_code=~\"4..|5..\", endpoint=\"$endpoint\"}[1h]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$endpoint Beacon Node Error QPH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "$endpoint Relay P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "$endpoint Relay P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PF1584C945D1A144E" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(cb_pbs_relay_latency_bucket{endpoint=\"$endpoint\"}[1m])) by (le, relay_id))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "$endpoint Relay P99", + "type": "timeseries" + } + ], + "refresh": "5m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "description": "BuilderAPI endpoint", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "endpoint", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "get_header", + "value": "get_header" + }, + { + "selected": false, + "text": "submit_blinded_block", + "value": "submit_blinded_block" + }, + { + "selected": false, + "text": "register_validator", + "value": "register_validator" + } + ], + "query": "get_header, submit_blinded_block, register_validator", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PBS Metrics", + "uid": "PF1584C945D1A144E", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/dashboards.yml b/grafana/dashboards/dashboards.yml new file mode 100644 index 00000000..f9f90c62 --- /dev/null +++ b/grafana/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + folderUid: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/grafana/datasources/datasources.yml b/grafana/datasources/datasources.yml new file mode 100644 index 00000000..de154426 --- /dev/null +++ b/grafana/datasources/datasources.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: cb-prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://cb_prometheus:9090 + isDefault: true + editable: true