diff --git a/bin/millau/node/src/chain_spec.rs b/bin/millau/node/src/chain_spec.rs index 3e812ca9aa..66ab997aa7 100644 --- a/bin/millau/node/src/chain_spec.rs +++ b/bin/millau/node/src/chain_spec.rs @@ -173,8 +173,10 @@ fn endowed_accounts() -> Vec { get_account_id_from_seed::("Rialto.MessagesSender"), // Accounts, used by RialtoParachain<>Millau bridge get_account_id_from_seed::(RIALTO_PARACHAIN_MESSAGES_PALLET_OWNER), - get_account_id_from_seed::("RialtoParachain.HeadersAndMessagesRelay"), - get_account_id_from_seed::("RialtoParachain.RialtoHeadersRelay"), + get_account_id_from_seed::("RialtoParachain.HeadersAndMessagesRelay1"), + get_account_id_from_seed::("RialtoParachain.HeadersAndMessagesRelay2"), + get_account_id_from_seed::("RialtoParachain.RialtoHeadersRelay1"), + get_account_id_from_seed::("RialtoParachain.RialtoHeadersRelay2"), get_account_id_from_seed::("RialtoParachain.MessagesSender"), ] .into_iter() diff --git a/bin/rialto-parachain/node/src/chain_spec.rs b/bin/rialto-parachain/node/src/chain_spec.rs index 6873730c19..4a7865b185 100644 --- a/bin/rialto-parachain/node/src/chain_spec.rs +++ b/bin/rialto-parachain/node/src/chain_spec.rs @@ -98,7 +98,8 @@ fn endowed_accounts() -> Vec { get_account_id_from_seed::("Ferdie//stash"), // Accounts, used by RialtoParachain<>Millau bridge get_account_id_from_seed::(MILLAU_MESSAGES_PALLET_OWNER), - get_account_id_from_seed::("Millau.HeadersAndMessagesRelay"), + get_account_id_from_seed::("Millau.HeadersAndMessagesRelay1"), + get_account_id_from_seed::("Millau.HeadersAndMessagesRelay2"), get_account_id_from_seed::("Millau.MessagesSender"), ] .into_iter() diff --git a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-millau-to-rialto-parachain-messages-dashboard.json b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-millau-to-rialto-parachain-messages-dashboard.json index 42af6f1479..b20348cc51 100644 --- a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-millau-to-rialto-parachain-messages-dashboard.json +++ b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-millau-to-rialto-parachain-messages-dashboard.json @@ -1,911 +1,910 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ + "annotations": { + "list": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_target_block_number", - "instant": false, - "interval": "", - "legendFormat": "At RialtoParachain", - "refId": "A" - }, - { - "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_target_at_source_block_number", - "instant": false, - "interval": "", - "legendFormat": "At Millau", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Best finalized RialtoParachain headers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_target_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "instant": false, + "interval": "", + "legendFormat": "At RialtoParachain", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_target_at_source_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "instant": false, + "interval": "", + "legendFormat": "At Millau", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Best finalized RialtoParachain headers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_source_block_number", - "interval": "", - "legendFormat": "At Millau", - "refId": "A" - }, - { - "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_source_at_target_block_number", - "interval": "", - "legendFormat": "At RialtoParachain", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Best finalized Millau headers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "lt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "Millau -> RialtoParachain messages are not detected by relay", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(label_replace(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=~\"source_latest_generated|target_latest_received\"}, \"type\", \"Latest message sent from RialtoParachain\", \"type\", \"source_latest_generated\"), \"type\", \"Latest RialtoParachain message received by Millau\", \"type\", \"target_latest_received\")", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - }, - { - "expr": "increase(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"source_latest_generated\"}[10m]) OR on() vector(0)", - "hide": true, - "interval": "", - "legendFormat": "Messages generated in last 5 minutes (Millau -> RialtoParachain, 00000000)", - "refId": "B" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "lt", - "value": 1, - "yaxis": "left" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Delivery race (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_source_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "interval": "", + "legendFormat": "At Millau", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "Millau_to_RialtoParachain_MessageLane_00000000_best_source_at_target_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "interval": "", + "legendFormat": "At RialtoParachain", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Best finalized Millau headers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "expr": "label_replace(label_replace(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=~\"source_latest_confirmed|target_latest_received\"}, \"type\", \"Latest message confirmed by RialtoParachain to Millau\", \"type\", \"source_latest_confirmed\"), \"type\", \"Latest RialtoParachain message received by Millau\", \"type\", \"target_latest_received\")", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" + "evaluator": { + "params": [ + 1 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "max" + }, + "type": "query" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Confirmations race (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "Millau -> RialtoParachain messages are not detected by relay", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(label_replace(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=~\"source_latest_generated|target_latest_received\"}, \"type\", \"Latest message sent from RialtoParachain\", \"type\", \"source_latest_generated\"), \"type\", \"Latest RialtoParachain message received by Millau\", \"type\", \"target_latest_received\")", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "increase(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_generated\"}[10m]) OR on() vector(0)", + "hide": true, + "interval": "", + "legendFormat": "Messages generated in last 5 minutes (Millau -> RialtoParachain, 00000000)", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1, + "yaxis": "left" } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delivery race (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "lt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "7m", - "frequency": "1m", - "handler": 1, - "name": "Messages from Millau to RialtoParachain are not being delivered", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"source_latest_generated\"}[2m])) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]))", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Undelivered messages at RialtoParachain", - "refId": "A" - }, - { - "expr": "increase(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[5m]) OR on() vector(0)", - "interval": "", - "legendFormat": "Millau Messages delivered to RialtoParachain in last 5m", - "refId": "B" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "lt", - "value": 1 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Delivery race lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "min" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "Too many unconfirmed messages (Millau -> RialtoParachain)", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 20 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(label_replace(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=~\"source_latest_confirmed|target_latest_received\"}, \"type\", \"Latest message confirmed by RialtoParachain to Millau\", \"type\", \"source_latest_confirmed\"), \"type\", \"Latest RialtoParachain message received by Millau\", \"type\", \"target_latest_received\")", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Confirmations race (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]) OR on() vector(0)) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m]) OR on() vector(0))", - "interval": "", - "legendFormat": "Unconfirmed messages at Millau", - "refId": "A" - } - ], - "thresholds": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 10 + "evaluator": { + "params": [ + 1 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "sum" + }, + "type": "query" } ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Confirmations race lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "executionErrorState": "alerting", + "for": "7m", + "frequency": "1m", + "handler": 1, + "name": "Messages from Millau to RialtoParachain are not being delivered", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_generated\"}[2m])) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Undelivered messages at RialtoParachain", + "refId": "A" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "expr": "increase(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[5m]) OR on() vector(0)", + "interval": "", + "legendFormat": "Millau Messages delivered to RialtoParachain in last 5m", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delivery race lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "evaluator": { + "params": [ + 50 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "min" + }, + "type": "query" } ], - "yaxis": { - "align": false, - "alignLevel": null + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "Too many unconfirmed messages (Millau -> RialtoParachain)", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]) OR on() vector(0)) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m]) OR on() vector(0))", + "interval": "", + "legendFormat": "Unconfirmed messages at Millau", + "refId": "A" } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Confirmations race lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 10 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "min" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "Rewards are not being confirmed (Millau -> RialtoParachain messages)", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 20 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m])) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_confirmed\"}[2m]))", - "interval": "", - "legendFormat": "Unconfirmed rewards at RialtoParachain", - "refId": "A" - }, - { - "expr": "(scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m]) OR on() vector(0)) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_confirmed\"}[2m]) OR on() vector(0))) * (max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]) > bool min_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]))", - "interval": "", - "legendFormat": "Unconfirmed rewards at Millau->RaltoParachain (zero if messages are not being delivered to RialtoParachain)", - "refId": "B" - } - ], - "thresholds": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 10, - "yaxis": "left" + "evaluator": { + "params": [ + 10 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "min" + }, + "type": "query" } ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Reward lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "Rewards are not being confirmed (Millau -> RialtoParachain messages)", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m])) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_confirmed\"}[2m]))", + "interval": "", + "legendFormat": "Unconfirmed rewards at RialtoParachain", + "refId": "A" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "expr": "(scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m]) OR on() vector(0)) - scalar(max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_confirmed\"}[2m]) OR on() vector(0))) * (max_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]) > bool min_over_time(Millau_to_RialtoParachain_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]))", + "interval": "", + "legendFormat": "Unconfirmed rewards at Millau->RaltoParachain (zero if messages are not being delivered to RialtoParachain)", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Reward lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null } - ], - "refresh": "5s", - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Millau to RialtoParachain Message Sync Dashboard", - "uid": "C61e-797z", - "version": 1 - } \ No newline at end of file + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Millau to RialtoParachain Message Sync Dashboard", + "uid": "C61e-797z", + "version": 1 +} diff --git a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-rialto-parachain-to-millau-messages-dashboard.json b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-rialto-parachain-to-millau-messages-dashboard.json index 89de7d6055..064436145d 100644 --- a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-rialto-parachain-to-millau-messages-dashboard.json +++ b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/relay-rialto-parachain-to-millau-messages-dashboard.json @@ -1,909 +1,908 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 1, - "links": [], - "panels": [ + "annotations": { + "list": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_target_block_number", - "instant": false, - "interval": "", - "legendFormat": "At Millau", - "refId": "A" - }, - { - "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_target_at_source_block_number", - "instant": false, - "interval": "", - "legendFormat": "At RialtoParachain", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Best finalized Millau headers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_target_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "instant": false, + "interval": "", + "legendFormat": "At Millau", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_target_at_source_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "instant": false, + "interval": "", + "legendFormat": "At RialtoParachain", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Best finalized Millau headers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_source_block_number", - "interval": "", - "legendFormat": "At RialtoParachain", - "refId": "A" - }, - { - "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_source_at_target_block_number", - "interval": "", - "legendFormat": "At Millau", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Best finalized RialtoParachain headers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "lt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "RialtoParachain -> Millau messages are not detected by relay", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(label_replace(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=~\"source_latest_generated|target_latest_received\"}, \"type\", \"Latest message sent from Millau\", \"type\", \"source_latest_generated\"), \"type\", \"Latest Millau message received by RialtoParachain\", \"type\", \"target_latest_received\")", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - }, - { - "expr": "increase(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"source_latest_generated\"}[10m]) OR on() vector(0)", - "hide": true, - "interval": "", - "legendFormat": "Messages generated in last 5 minutes (RialtoParachain -> Millau, 00000000)", - "refId": "B" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "lt", - "value": 1 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Delivery race (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_source_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "interval": "", + "legendFormat": "At RialtoParachain", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "RialtoParachain_to_Millau_MessageLane_00000000_best_source_at_target_block_number{instance=\"relay-millau-rialto-parachain-1:9616\"}", + "interval": "", + "legendFormat": "At Millau", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Best finalized RialtoParachain headers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "expr": "label_replace(label_replace(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=~\"source_latest_confirmed|target_latest_received\"}, \"type\", \"Latest message confirmed by Millau to RialtoParachain\", \"type\", \"source_latest_confirmed\"), \"type\", \"Latest Millau message received by RialtoParachain\", \"type\", \"target_latest_received\")", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" + "evaluator": { + "params": [ + 1 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "max" + }, + "type": "query" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Confirmations race (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "RialtoParachain -> Millau messages are not detected by relay", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(label_replace(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=~\"source_latest_generated|target_latest_received\"}, \"type\", \"Latest message sent from Millau\", \"type\", \"source_latest_generated\"), \"type\", \"Latest Millau message received by RialtoParachain\", \"type\", \"target_latest_received\")", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "increase(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_generated\"}[10m]) OR on() vector(0)", + "hide": true, + "interval": "", + "legendFormat": "Messages generated in last 5 minutes (RialtoParachain -> Millau, 00000000)", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1 } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delivery race (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "lt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "7m", - "frequency": "1m", - "handler": 1, - "name": "Messages from RialtoParachain to Millau are not being delivered", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"source_latest_generated\"}[2m])) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]))", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Undelivered messages at Millau", - "refId": "A" - }, - { - "expr": "increase(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[5m]) OR on() vector(0)", - "interval": "", - "legendFormat": "RialtoParachain Messages delivered to Millau in last 5m", - "refId": "B" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "lt", - "value": 1 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Delivery race lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "min" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "Too many unconfirmed messages (RialtoParachain -> Millau)", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 20 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(label_replace(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=~\"source_latest_confirmed|target_latest_received\"}, \"type\", \"Latest message confirmed by Millau to RialtoParachain\", \"type\", \"source_latest_confirmed\"), \"type\", \"Latest Millau message received by RialtoParachain\", \"type\", \"target_latest_received\")", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Confirmations race (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]) OR on() vector(0)) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m]) OR on() vector(0))", - "interval": "", - "legendFormat": "Unconfirmed messages at RialtoParachain", - "refId": "A" - } - ], - "thresholds": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 10 + "evaluator": { + "params": [ + 1 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "sum" + }, + "type": "query" } ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Confirmations race lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "executionErrorState": "alerting", + "for": "7m", + "frequency": "1m", + "handler": 1, + "name": "Messages from RialtoParachain to Millau are not being delivered", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_generated\"}[2m])) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Undelivered messages at Millau", + "refId": "A" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "expr": "increase(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[5m]) OR on() vector(0)", + "interval": "", + "legendFormat": "RialtoParachain Messages delivered to Millau in last 5m", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delivery race lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "evaluator": { + "params": [ + 50 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "min" + }, + "type": "query" } ], - "yaxis": { - "align": false, - "alignLevel": null + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "Too many unconfirmed messages (RialtoParachain -> Millau)", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]) OR on() vector(0)) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m]) OR on() vector(0))", + "interval": "", + "legendFormat": "Unconfirmed messages at RialtoParachain", + "refId": "A" } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Confirmations race lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 10 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "min" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "20m", - "frequency": "1m", - "handler": 1, - "name": "Rewards are not being confirmed (RialtoParachain -> Millau messages)", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 20 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m])) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_confirmed\"}[2m]))", - "interval": "", - "legendFormat": "Unconfirmed rewards at Millau", - "refId": "A" - }, - { - "expr": "(scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"source_latest_confirmed\"}[2m]) OR on() vector(0)) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_confirmed\"}[2m]) OR on() vector(0))) * (max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]) > bool min_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{type=\"target_latest_received\"}[2m]))", - "interval": "", - "legendFormat": "Unconfirmed rewards at Millau (zero if messages are not being delivered to Millau)", - "refId": "B" - } - ], - "thresholds": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 10 + "evaluator": { + "params": [ + 10 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "min" + }, + "type": "query" } ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Reward lags (00000000)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "executionErrorState": "alerting", + "for": "20m", + "frequency": "1m", + "handler": 1, + "name": "Rewards are not being confirmed (RialtoParachain -> Millau messages)", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m])) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_confirmed\"}[2m]))", + "interval": "", + "legendFormat": "Unconfirmed rewards at Millau", + "refId": "A" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "expr": "(scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"source_latest_confirmed\"}[2m]) OR on() vector(0)) - scalar(max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_confirmed\"}[2m]) OR on() vector(0))) * (max_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]) > bool min_over_time(RialtoParachain_to_Millau_MessageLane_00000000_lane_state_nonces{instance=\"relay-millau-rialto-parachain-1:9616\",type=\"target_latest_received\"}[2m]))", + "interval": "", + "legendFormat": "Unconfirmed rewards at Millau (zero if messages are not being delivered to Millau)", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Reward lags (00000000)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null } - ], - "refresh": "5s", - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "RialtoParachain to Millau Message Sync Dashboard", - "uid": "-l61a7r7k", - "version": 1 - } \ No newline at end of file + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "RialtoParachain to Millau Message Sync Dashboard", + "uid": "-l61a7r7k", + "version": 1 +} diff --git a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/rialto-parachain-millau-maintenance-dashboard.json b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/rialto-parachain-millau-maintenance-dashboard.json index 76330ea085..a8305e84e4 100644 --- a/deployments/bridges/rialto-parachain-millau/dashboard/grafana/rialto-parachain-millau-maintenance-dashboard.json +++ b/deployments/bridges/rialto-parachain-millau/dashboard/grafana/rialto-parachain-millau-maintenance-dashboard.json @@ -66,7 +66,7 @@ "targets": [ { "exemplar": true, - "expr": "rialtoparachain_storage_proof_overhead{instance='relay-millau-rialto-parachain:9616'}", + "expr": "rialtoparachain_storage_proof_overhead{instance='relay-millau-rialto-parachain-1:9616'}", "interval": "", "legendFormat": "Actual overhead", "refId": "A" @@ -170,14 +170,14 @@ "targets": [ { "exemplar": true, - "expr": "polkadot_to_base_conversion_rate{instance='relay-millau-rialto-parachain:9616'} / kusama_to_base_conversion_rate{instance='relay-millau-rialto-parachain:9616'}", + "expr": "polkadot_to_base_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'} / kusama_to_base_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'}", "interval": "", "legendFormat": "Outside of runtime (actually Polkadot -> Kusama)", "refId": "A" }, { "exemplar": true, - "expr": "Millau_RialtoParachain_to_Millau_conversion_rate{instance='relay-millau-rialto-parachain:9616'}", + "expr": "Millau_RialtoParachain_to_Millau_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'}", "hide": false, "interval": "", "legendFormat": "At runtime", @@ -274,7 +274,7 @@ "targets": [ { "exemplar": true, - "expr": "millau_storage_proof_overhead{instance='relay-millau-rialto-parachain:9616'}", + "expr": "millau_storage_proof_overhead{instance='relay-millau-rialto-parachain-1:9616'}", "interval": "", "legendFormat": "Actual overhead", "refId": "A" @@ -378,14 +378,14 @@ "targets": [ { "exemplar": true, - "expr": "kusama_to_base_conversion_rate{instance='relay-millau-rialto-parachain:9616'} / polkadot_to_base_conversion_rate{instance='relay-millau-rialto-parachain:9616'}", + "expr": "kusama_to_base_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'} / polkadot_to_base_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'}", "interval": "", "legendFormat": "Outside of runtime (actually Kusama -> Polkadot)", "refId": "A" }, { "exemplar": true, - "expr": "RialtoParachain_Millau_to_RialtoParachain_conversion_rate{instance='relay-millau-rialto-parachain:9616'}", + "expr": "RialtoParachain_Millau_to_RialtoParachain_conversion_rate{instance='relay-millau-rialto-parachain-1:9616'}", "hide": false, "interval": "", "legendFormat": "At runtime", @@ -559,19 +559,19 @@ "steppedLine": false, "targets": [ { - "expr": "at_RialtoParachain_relay_MillauHeaders_balance", + "expr": "at_RialtoParachain_relay_MillauHeaders_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-Millau headers relay account balance", "refId": "A" }, { - "expr": "at_RialtoParachain_relay_MillauMessages_balance", + "expr": "at_RialtoParachain_relay_MillauMessages_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-Millau messages relay account balance", "refId": "B" }, { - "expr": "at_RialtoParachain_relay_MillauMessagesPalletOwner_balance", + "expr": "at_RialtoParachain_relay_MillauMessagesPalletOwner_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-Millau messages pallet owner account balance", "refId": "C" @@ -752,25 +752,25 @@ "steppedLine": false, "targets": [ { - "expr": "at_Millau_relay_RialtoHeaders_balance{instance=\"relay-millau-rialto-parachain:9616\"}", + "expr": "at_Millau_relay_RialtoHeaders_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-Rialto headers relay account balance", "refId": "A" }, { - "expr": "at_Millau_relay_RialtoParachainMessages_balance", + "expr": "at_Millau_relay_RialtoParachainMessages_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-RialtoParachain messages relay account balance", "refId": "B" }, { - "expr": "at_Millau_relay_RialtoParachains_balance", + "expr": "at_Millau_relay_RialtoParachains_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-Rialto parachains relay account balance", "refId": "C" }, { - "expr": "at_Millau_relay_RialtoParachainMessagesPalletOwner_balance", + "expr": "at_Millau_relay_RialtoParachainMessagesPalletOwner_balance{instance=\"relay-millau-rialto-parachain-1:9616\"}", "interval": "", "legendFormat": "With-RialtoParachain messages pallet owner account balance", "refId": "D" @@ -905,7 +905,7 @@ "steppedLine": false, "targets": [ { - "expr": "RialtoParachain_to_Millau_MessageLane_00000000_is_source_and_source_at_target_using_different_forks OR on() vector(0)", + "expr": "RialtoParachain_to_Millau_MessageLane_00000000_is_source_and_source_at_target_using_different_forks{instance=\"relay-millau-rialto-parachain-1:9616\"} OR on() vector(0)", "instant": false, "interval": "", "legendFormat": "On different forks?", @@ -1041,7 +1041,7 @@ "steppedLine": false, "targets": [ { - "expr": "Millau_to_RialtoParachain_MessageLane_00000000_is_source_and_source_at_target_using_different_forks OR on() vector(0)", + "expr": "Millau_to_RialtoParachain_MessageLane_00000000_is_source_and_source_at_target_using_different_forks{instance=\"relay-millau-rialto-parachain-1:9616\"} OR on() vector(0)", "interval": "", "legendFormat": "On different forks?", "refId": "A" diff --git a/deployments/bridges/rialto-parachain-millau/dashboard/prometheus/targets.yml b/deployments/bridges/rialto-parachain-millau/dashboard/prometheus/targets.yml index 68255db0c6..ba683e3479 100644 --- a/deployments/bridges/rialto-parachain-millau/dashboard/prometheus/targets.yml +++ b/deployments/bridges/rialto-parachain-millau/dashboard/prometheus/targets.yml @@ -1,2 +1,3 @@ - targets: - - relay-millau-rialto-parachain:9616 + - relay-millau-rialto-parachain-1:9616 + - relay-millau-rialto-parachain-2:9616 diff --git a/deployments/bridges/rialto-parachain-millau/docker-compose.yml b/deployments/bridges/rialto-parachain-millau/docker-compose.yml index e20af1f817..a90e225eb1 100644 --- a/deployments/bridges/rialto-parachain-millau/docker-compose.yml +++ b/deployments/bridges/rialto-parachain-millau/docker-compose.yml @@ -1,4 +1,4 @@ -# Exposed ports: 10816, 10916, 11016, 11017 +# Exposed ports: 10816, 10916, 11016, 11017, 11018 version: '3.5' services: @@ -18,7 +18,7 @@ services: LETSENCRYPT_HOST: wss.millau.brucke.link LETSENCRYPT_EMAIL: admin@parity.io - relay-millau-rialto-parachain: &sub-bridge-relay + relay-millau-rialto-parachain-1: &sub-bridge-relay image: paritytech/substrate-relay entrypoint: /entrypoints/relay-millau-rialto-parachain-entrypoint.sh volumes: @@ -37,21 +37,30 @@ services: - rialto-parachain-collator-bob - rialto-parachain-collator-charlie - relay-messages-millau-to-rialto-parachain-generator: + relay-millau-rialto-parachain-2: <<: *sub-bridge-relay + environment: + RUST_LOG: rpc=trace,bridge=trace + EXT_MILLAU_RELAY_ACCOUNT: //RialtoParachain.HeadersAndMessagesRelay2 + EXT_MILLAU_RELAY_ACCOUNT_HEADERS_OVERRIDE: //RialtoParachain.RialtoHeadersRelay2 + EXT_RIALTO_PARACHAIN_RELAY_ACCOUNT: //Millau.HeadersAndMessagesRelay2 ports: - "10916:9616" + relay-messages-millau-to-rialto-parachain-generator: + <<: *sub-bridge-relay + ports: + - "11016:9616" entrypoint: /entrypoints/relay-messages-to-rialto-parachain-generator-entrypoint.sh depends_on: - - relay-millau-rialto-parachain + - relay-millau-rialto-parachain-1 relay-messages-rialto-parachain-to-millau-generator: <<: *sub-bridge-relay entrypoint: /entrypoints/relay-messages-to-millau-generator-entrypoint.sh ports: - - "11016:9616" + - "11017:9616" depends_on: - - relay-millau-rialto-parachain + - relay-millau-rialto-parachain-1 relay-messages-millau-to-rialto-parachain-resubmitter: <<: *sub-bridge-relay @@ -59,7 +68,7 @@ services: RUST_LOG: bridge=trace entrypoint: /entrypoints/relay-messages-to-rialto-parachain-resubmitter-entrypoint.sh ports: - - "11017:9616" + - "11018:9616" depends_on: - relay-messages-millau-to-rialto-parachain-generator diff --git a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh index 45670ca107..f2732e82f4 100755 --- a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh +++ b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh @@ -3,6 +3,10 @@ set -xeu sleep 15 +MILLAU_RELAY_ACCOUNT=${EXT_MILLAU_RELAY_ACCOUNT:-//RialtoParachain.HeadersAndMessagesRelay1} +MILLAU_RELAY_ACCOUNT_HEADERS_OVERRIDE=${EXT_MILLAU_RELAY_ACCOUNT_HEADERS_OVERRIDE:-//RialtoParachain.RialtoHeadersRelay1} +RIALTO_PARACHAIN_RELAY_ACCOUNT=${EXT_RIALTO_PARACHAIN_RELAY_ACCOUNT:-//Millau.HeadersAndMessagesRelay1} + /home/user/substrate-relay init-bridge millau-to-rialto-parachain \ --source-host millau-node-alice \ --source-port 9944 \ @@ -23,13 +27,13 @@ sleep 6 /home/user/substrate-relay relay-headers-and-messages millau-rialto-parachain \ --millau-host millau-node-alice \ --millau-port 9944 \ - --millau-signer //RialtoParachain.HeadersAndMessagesRelay \ - --rialto-headers-to-millau-signer //RialtoParachain.RialtoHeadersRelay \ + --millau-signer $MILLAU_RELAY_ACCOUNT \ + --rialto-headers-to-millau-signer $MILLAU_RELAY_ACCOUNT_HEADERS_OVERRIDE \ --millau-messages-pallet-owner=//RialtoParachain.MessagesOwner \ --millau-transactions-mortality=64 \ --rialto-parachain-host rialto-parachain-collator-charlie \ --rialto-parachain-port 9944 \ - --rialto-parachain-signer //Millau.HeadersAndMessagesRelay \ + --rialto-parachain-signer $RIALTO_PARACHAIN_RELAY_ACCOUNT \ --rialto-parachain-messages-pallet-owner=//Millau.MessagesOwner \ --rialto-parachain-transactions-mortality=64 \ --rialto-host rialto-node-alice \ diff --git a/deployments/bridges/westend-millau/docker-compose.yml b/deployments/bridges/westend-millau/docker-compose.yml index 91d1c1fd07..193030191b 100644 --- a/deployments/bridges/westend-millau/docker-compose.yml +++ b/deployments/bridges/westend-millau/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.5' services: relay-headers-westend-to-millau-1: - image: paritytech/substrate-relay + image: local/substrate-relay entrypoint: /entrypoints/relay-headers-westend-to-millau-entrypoint.sh volumes: - ./bridges/westend-millau/entrypoints:/entrypoints @@ -15,7 +15,7 @@ services: - millau-node-alice relay-headers-westend-to-millau-2: - image: paritytech/substrate-relay + image: local/substrate-relay entrypoint: /entrypoints/relay-headers-westend-to-millau-entrypoint.sh volumes: - ./bridges/westend-millau/entrypoints:/entrypoints @@ -28,7 +28,7 @@ services: - millau-node-alice relay-parachains-westend-to-millau-1: - image: paritytech/substrate-relay + image: local/substrate-relay entrypoint: /entrypoints/relay-parachains-westend-to-millau-entrypoint.sh volumes: - ./bridges/westend-millau/entrypoints:/entrypoints @@ -40,7 +40,7 @@ services: - millau-node-alice relay-parachains-westend-to-millau-2: - image: paritytech/substrate-relay + image: local/substrate-relay entrypoint: /entrypoints/relay-parachains-westend-to-millau-entrypoint.sh volumes: - ./bridges/westend-millau/entrypoints:/entrypoints diff --git a/scripts/dump-logs.sh b/scripts/dump-logs.sh index cc0cd2c7ad..51e8518872 100755 --- a/scripts/dump-logs.sh +++ b/scripts/dump-logs.sh @@ -24,7 +24,8 @@ SERVICES=(\ deployments_relay-parachains-westend-to-millau-1_2 \ deployments_relay-messages-millau-to-rialto-parachain-generator_1 \ deployments_relay-messages-rialto-parachain-to-millau-generator_1 \ - deployments_relay-millau-rialto-parachain_1 \ + deployments_relay-millau-rialto-parachain-1_1 \ + deployments_relay-millau-rialto-parachain-2_1 \ deployments_rialto-node-alice_1 \ deployments_rialto-node-bob_1 \ deployments_millau-node-alice_1 \