From f0d432e6d358732c49db60c316ce886506aae491 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Thu, 19 Jun 2025 20:50:39 +0530 Subject: [PATCH 1/2] HDDS-13240. Add newly added metrics into grafana dashboard. --- .../dashboards/Ozone - DeleteKey Metrics.json | 406 +++++++++++++++++- .../Ozone - DeleteKeyProgress Metrics.json | 2 +- .../ozone/om/DeletingServiceMetrics.java | 2 +- 3 files changed, 395 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json index d026dc82a370..5a015dd34b44 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json @@ -121,7 +121,7 @@ "useBackend": false } ], - "title": "No. of Failed Key Deletes", + "title": "No. of delete request failed", "type": "timeseries" }, { @@ -309,7 +309,7 @@ "useBackend": false } ], - "title": "No. of Successful Key Deletes", + "title": "No. of delete request received", "type": "timeseries" }, { @@ -1077,7 +1077,7 @@ "useBackend": false } ], - "title": "KeyDeletingService: Keys processed", + "title": "KeyDeletingService: No. of keys sent to SCM for deletion", "type": "timeseries" }, { @@ -1171,7 +1171,7 @@ "useBackend": false } ], - "title": "KeyDeletingService: Keys sent for purge", + "title": "KeyDeletingService: Keys purged from OM", "type": "timeseries" }, { @@ -1265,7 +1265,7 @@ "useBackend": false } ], - "title": "DirectoryDeletingService: Dirs", + "title": "DirectoryDeletingService: No. of Dirs deleted", "type": "timeseries" }, { @@ -1361,7 +1361,7 @@ "useBackend": false } ], - "title": "DirectoryDeletingService: Files", + "title": "DirectoryDeletingService: No. of Files moved to deletedTable", "type": "timeseries" }, { @@ -1549,7 +1549,7 @@ "useBackend": false } ], - "title": "PurgeRequests: Keys", + "title": "PurgeRequests: Keys Purged", "type": "timeseries" }, { @@ -1927,7 +1927,7 @@ "useBackend": false } ], - "title": "OpenKeyDeletion: Keys deleted", + "title": "Number of OpenKeys Deleted", "type": "timeseries" }, { @@ -2413,7 +2413,7 @@ "useBackend": false } ], - "title": "No. of Successful Key Deletes", + "title": "Number of keys received for deletion from OM", "type": "timeseries" }, { @@ -2787,6 +2787,100 @@ ], "title": "Percentage of successful delete transactions", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "No. of blocks", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "scm_performance_metrics_delete_key_blocks_success", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of blocks received for deletion from OM", + "type": "timeseries" } ], "title": "SCM: Summary", @@ -3177,7 +3271,7 @@ "useBackend": false } ], - "title": "No. of deletion transactions created", + "title": "Number of transactions(group of blocks) created", "type": "timeseries" }, { @@ -3271,7 +3365,7 @@ "useBackend": false } ], - "title": "No. of deletion transactions sent", + "title": "Number of transactions(group of blocks) sent for deletion to DN", "type": "timeseries" }, { @@ -3365,7 +3459,7 @@ "useBackend": false } ], - "title": "No. of deletion transactions completed", + "title": "Number of transactions(group of blocks) completed", "type": "timeseries" }, { @@ -3450,7 +3544,7 @@ { "disableTextWrap": false, "editorMode": "builder", - "expr": "scm_block_deleting_service_num_block_deletion_transaction_success", + "expr": "scm_block_deleting_service_num_block_deletion_transaction_success_on_datanodes", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}}", @@ -3651,6 +3745,292 @@ ], "title": "No. of deletion transactions skipped", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "No. of blocks", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1313 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "scm_block_deleting_service_blocks_sent", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{datanode}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "No. of blocks sent to each DN for deletion", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "No. of blocks", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1313 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "scm_block_deleting_service_num_block_added_for_deletion_to_dn", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of blocks sent for deletion to DN", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "No. of commands", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1313 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "scm_block_deleting_service_commands_timeout", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{datanode}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of commands timeout from SCM to DN", + "type": "timeseries" } ], "title": "SCM: Deletion commands and transactions", diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKeyProgress Metrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKeyProgress Metrics.json index 687eb7f4a3d2..e90e5a1ccc9d 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKeyProgress Metrics.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKeyProgress Metrics.json @@ -1707,6 +1707,6 @@ }, "timepicker": {}, "timezone": "", - "title": "Ozone DeleteKey – Lightweight Dashboard", + "title": "Deletion Progress of dirs and keys in Ozone", "weekStart": "" } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java index baa4a34e774b..f343f0c012c6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java @@ -38,7 +38,7 @@ public final class DeletingServiceMetrics { /* * Total directory deletion metrics across all iterations of DirectoryDeletingService since last restart. */ - @Metric("Total no. of deleted directories sent for purge") + @Metric("Total no. of directories deleted") private MutableGaugeLong numDirsSentForPurge; @Metric("Total no. of sub-directories sent for purge") private MutableGaugeLong numSubDirsSentForPurge; From 7008dca80d048091369ceeaafcc5f04f1c36f7cf Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 24 Jun 2025 11:31:26 +0530 Subject: [PATCH 2/2] Addressed comments. --- .../common/grafana/dashboards/Ozone - DeleteKey Metrics.json | 4 ++-- .../org/apache/hadoop/ozone/om/DeletingServiceMetrics.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json index 5a015dd34b44..764dabd85916 100644 --- a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json +++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DeleteKey Metrics.json @@ -1171,7 +1171,7 @@ "useBackend": false } ], - "title": "KeyDeletingService: Keys purged from OM", + "title": "KeyDeletingService: No. of keys to be purged", "type": "timeseries" }, { @@ -1549,7 +1549,7 @@ "useBackend": false } ], - "title": "PurgeRequests: Keys Purged", + "title": "PurgeRequests: No. of keys purged", "type": "timeseries" }, { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java index f343f0c012c6..baa4a34e774b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java @@ -38,7 +38,7 @@ public final class DeletingServiceMetrics { /* * Total directory deletion metrics across all iterations of DirectoryDeletingService since last restart. */ - @Metric("Total no. of directories deleted") + @Metric("Total no. of deleted directories sent for purge") private MutableGaugeLong numDirsSentForPurge; @Metric("Total no. of sub-directories sent for purge") private MutableGaugeLong numSubDirsSentForPurge;