diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json index c2d22cc0..de81fb91 100644 --- a/etc/grafana/dashboards/foyer.json +++ b/etc/grafana/dashboards/foyer.json @@ -1 +1 @@ -{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":22,"title":"Hybrid","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":23,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - hybrid - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":24,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - hybrid - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - hybrid - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - hybrid - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - hybrid - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_hybrid_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - hybrid - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":25,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_hybrid_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - hybrid - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"gridPos":{"h":1,"w":24,"x":0,"y":17},"id":14,"title":"Memory","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"id":13,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - memory - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"id":15,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","instant":false,"legendFormat":"{{name}} - memory - usage (bytes)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","hide":true,"instant":false,"legendFormat":"{{name}} - memory - usage (count)","range":true,"refId":"B"}],"title":"Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_memory_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - memory - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":34},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":35},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Inner Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":43},"id":16,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":43},"id":17,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_inner_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":51},"id":27,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type)","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":51},"id":28,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type) * on(name) group_left() foyer_storage_region_size_bytes","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":59},"id":18,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_storage_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - storage - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":67},"id":19,"panels":[],"title":"Storage (Disk)","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":68},"id":20,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - disk io - {{op}}","range":true,"refId":"A"}],"title":"Disk IO","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":68},"id":21,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - disk io - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - disk io - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - disk io - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - disk io - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_disk_io_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - disk io - {{op}}","range":true,"refId":"E"}],"title":"Disk IO Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":76},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"}],"refresh":"5s","schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timeRangeUpdatedDuringEditOrView":false,"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""} +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":22,"title":"Hybrid","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":23,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - hybrid - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":24,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - hybrid - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - hybrid - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - hybrid - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - hybrid - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_hybrid_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - hybrid - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":25,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_hybrid_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - hybrid - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"gridPos":{"h":1,"w":24,"x":0,"y":17},"id":14,"title":"Memory","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"id":13,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - memory - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"id":15,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","instant":false,"legendFormat":"{{name}} - memory - usage (bytes)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","hide":true,"instant":false,"legendFormat":"{{name}} - memory - usage (count)","range":true,"refId":"B"}],"title":"Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_memory_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - memory - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":34},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":35},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Inner Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":43},"id":16,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":43},"id":17,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_inner_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":51},"id":27,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type)","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":51},"id":28,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type) * on(name) group_left() foyer_storage_region_size_bytes","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":59},"id":18,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_storage_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - storage - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":59},"id":29,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_entry_serde_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_entry_serde_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Serde Duration","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":67},"id":19,"panels":[],"title":"Storage (Disk)","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":68},"id":20,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - disk io - {{op}}","range":true,"refId":"A"}],"title":"Disk IO","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":68},"id":21,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - disk io - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - disk io - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - disk io - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - disk io - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_disk_io_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - disk io - {{op}}","range":true,"refId":"E"}],"title":"Disk IO Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":76},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"}],"refresh":"5s","schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""} diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml index 1b8304b8..9036bd76 100644 --- a/foyer-bench/Cargo.toml +++ b/foyer-bench/Cargo.toml @@ -33,7 +33,7 @@ opentelemetry_sdk = { version = "0.24", features = [ parking_lot = "0.12" rand = "0.8.5" serde = { workspace = true } -serde_bytes = "0.11.14" +serde_bytes = "0.11.15" tokio = { workspace = true } tracing = "0.1" tracing-opentelemetry = { version = "0.25", optional = true } diff --git a/foyer-common/src/metrics.rs b/foyer-common/src/metrics.rs index 47b2f353..f1f81132 100644 --- a/foyer-common/src/metrics.rs +++ b/foyer-common/src/metrics.rs @@ -66,6 +66,9 @@ pub struct Metrics { pub storage_region_size_bytes: Gauge, + pub storage_entry_serialize_duration: Histogram, + pub storage_entry_deserialize_duration: Histogram, + /* hybrid cache metrics */ pub hybrid_insert: Counter, pub hybrid_hit: Counter, @@ -156,6 +159,10 @@ impl Metrics { let storage_region_size_bytes = gauge!(format!("foyer_storage_region_size_bytes"), "name" => name.to_string()); + let storage_entry_serialize_duration = + histogram!(format!("foyer_storage_entry_serde_duration"), "name" => name.to_string(), "op" => "serialize"); + let storage_entry_deserialize_duration = histogram!(format!("foyer_storage_entry_serde_duration"), "name" => name.to_string(), "op" => "deserialize"); + /* hybrid cache metrics */ let hybrid_insert = counter!(format!("foyer_hybrid_op_total"), "name" => name.to_string(), "op" => "insert"); @@ -210,6 +217,8 @@ impl Metrics { storage_region_clean, storage_region_evictable, storage_region_size_bytes, + storage_entry_serialize_duration, + storage_entry_deserialize_duration, hybrid_insert, hybrid_hit, diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs index 23de2d63..3c94491e 100644 --- a/foyer-storage/src/large/generic.rs +++ b/foyer-storage/src/large/generic.rs @@ -228,6 +228,7 @@ where &indexer, ®ion_manager, &tombstones, + metrics.clone(), config.user_runtime_handle.clone(), ) .await?; @@ -256,6 +257,7 @@ where flushers.clone(), stats.clone(), config.flush, + metrics.clone(), config.write_runtime_handle.clone(), ) .await @@ -346,6 +348,7 @@ where header.value_len as _, header.compression, Some(header.checksum), + &metrics, ) { Ok(res) => res, Err(e) => match e { @@ -499,7 +502,7 @@ mod tests { }, picker::utils::{FifoPicker, RejectAllPicker}, serde::EntrySerializer, - test_utils::BiasedPicker, + test_utils::{metrics_for_test, BiasedPicker}, IoBytesMut, TombstoneLogConfigBuilder, }; @@ -594,7 +597,14 @@ mod tests { fn enqueue(store: &GenericLargeStorage, RandomState>, entry: CacheEntry, RandomState>) { let mut buffer = IoBytesMut::new(); - let info = EntrySerializer::serialize(entry.key(), entry.value(), &Compression::None, &mut buffer).unwrap(); + let info = EntrySerializer::serialize( + entry.key(), + entry.value(), + &Compression::None, + &mut buffer, + metrics_for_test(), + ) + .unwrap(); let buffer = buffer.freeze(); store.enqueue(entry, buffer, info); } diff --git a/foyer-storage/src/large/reclaimer.rs b/foyer-storage/src/large/reclaimer.rs index 29238828..92e7fddb 100644 --- a/foyer-storage/src/large/reclaimer.rs +++ b/foyer-storage/src/large/reclaimer.rs @@ -14,7 +14,10 @@ use std::{fmt::Debug, future::Future, sync::Arc, time::Duration}; -use foyer_common::code::{HashBuilder, StorageKey, StorageValue}; +use foyer_common::{ + code::{HashBuilder, StorageKey, StorageValue}, + metrics::Metrics, +}; use futures::future::join_all; use itertools::Itertools; use tokio::{ @@ -53,6 +56,7 @@ impl Reclaimer { flushers: Vec>, stats: Arc, flush: bool, + metrics: Arc, runtime: Handle, ) -> Self where @@ -70,6 +74,7 @@ impl Reclaimer { reinsertion_picker, stats, flush, + metrics, wait_rx, runtime: runtime.clone(), }; @@ -108,6 +113,8 @@ where flush: bool, + metrics: Arc, + wait_rx: mpsc::UnboundedReceiver>, runtime: Handle, @@ -167,7 +174,7 @@ where tracing::debug!("[reclaimer]: Start reclaiming region {id}."); - let mut scanner = RegionScanner::new(region.clone()); + let mut scanner = RegionScanner::new(region.clone(), self.metrics.clone()); let mut picked_count = 0; let mut unpicked = vec![]; // The loop will ends when: diff --git a/foyer-storage/src/large/recover.rs b/foyer-storage/src/large/recover.rs index 6faa6d25..45470b3e 100644 --- a/foyer-storage/src/large/recover.rs +++ b/foyer-storage/src/large/recover.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use clap::ValueEnum; use foyer_common::code::{HashBuilder, StorageKey, StorageValue}; +use foyer_common::metrics::Metrics; use futures::future::try_join_all; use itertools::Itertools; @@ -61,6 +62,8 @@ pub enum RecoverMode { pub struct RecoverRunner; impl RecoverRunner { + // TODO(MrCroxx): use `expect` after `lint_reasons` is stable. + #[allow(clippy::too_many_arguments)] pub async fn run( config: &GenericLargeStorageConfig, regions: Range, @@ -68,6 +71,7 @@ impl RecoverRunner { indexer: &Indexer, region_manager: &RegionManager, tombstones: &[Tombstone], + metrics: Arc, runtime: Handle, ) -> Result<()> where @@ -81,9 +85,10 @@ impl RecoverRunner { let handles = regions.map(|id| { let semaphore = semaphore.clone(); let region = region_manager.region(id).clone(); + let metrics = metrics.clone(); runtime.spawn(async move { let permit = semaphore.acquire().await; - let res = RegionRecoverRunner::run(mode, region).await; + let res = RegionRecoverRunner::run(mode, region, metrics).await; drop(permit); res }) @@ -194,7 +199,7 @@ impl RecoverRunner { struct RegionRecoverRunner; impl RegionRecoverRunner { - async fn run(mode: RecoverMode, region: Region) -> Result> { + async fn run(mode: RecoverMode, region: Region, metrics: Arc) -> Result> { if mode == RecoverMode::None { return Ok(vec![]); } @@ -202,7 +207,7 @@ impl RegionRecoverRunner { let mut infos = vec![]; let id = region.id(); - let mut iter = RegionScanner::new(region); + let mut iter = RegionScanner::new(region, metrics); loop { let r = iter.next().await; match r { diff --git a/foyer-storage/src/large/scanner.rs b/foyer-storage/src/large/scanner.rs index cd6c4fe0..bd09a452 100644 --- a/foyer-storage/src/large/scanner.rs +++ b/foyer-storage/src/large/scanner.rs @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use foyer_common::{ bits, code::{StorageKey, StorageValue}, + metrics::Metrics, strict_assert, }; @@ -83,15 +86,17 @@ pub struct RegionScanner { region: Region, offset: u64, cache: CachedDeviceReader, + metrics: Arc, } impl RegionScanner { - pub fn new(region: Region) -> Self { + pub fn new(region: Region, metrics: Arc) -> Self { let cache = CachedDeviceReader::new(region.clone()); Self { region, offset: 0, cache, + metrics, } } @@ -216,6 +221,7 @@ impl RegionScanner { header.value_len as _, header.compression, Some(header.checksum), + &self.metrics, )?; self.step(&header).await; diff --git a/foyer-storage/src/serde.rs b/foyer-storage/src/serde.rs index fe63184e..de9bcb50 100644 --- a/foyer-storage/src/serde.rs +++ b/foyer-storage/src/serde.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{fmt::Debug, hash::Hasher}; +use std::{fmt::Debug, hash::Hasher, time::Instant}; use twox_hash::XxHash64; -use foyer_common::code::{StorageKey, StorageValue}; +use foyer_common::{ + code::{StorageKey, StorageValue}, + metrics::Metrics, +}; use crate::{ compress::Compression, @@ -51,11 +54,14 @@ impl EntrySerializer { value: &'a V, compression: &'a Compression, mut buffer: &'a mut IoBytesMut, + metrics: &Metrics, ) -> Result where K: StorageKey, V: StorageValue, { + let now = Instant::now(); + let mut cursor = buffer.len(); // serialize value @@ -85,6 +91,8 @@ impl EntrySerializer { bincode::serialize_into(&mut buffer, &key).map_err(Error::from)?; let key_len = buffer.len() - cursor; + metrics.storage_entry_serialize_duration.record(now.elapsed()); + Ok(KvInfo { key_len, value_len }) } } @@ -100,11 +108,14 @@ impl EntryDeserializer { value_len: usize, compression: Compression, checksum: Option, + metrics: &Metrics, ) -> Result<(K, V)> where K: StorageKey, V: StorageValue, { + let now = Instant::now(); + // deserialize value let buf = &buffer[..value_len]; let value = Self::deserialize_value(buf, compression)?; @@ -121,6 +132,8 @@ impl EntryDeserializer { } } + metrics.storage_entry_deserialize_duration.record(now.elapsed()); + Ok((key, value)) } diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs index dcccd1ae..82cfcb99 100644 --- a/foyer-storage/src/store.rs +++ b/foyer-storage/src/store.rs @@ -143,7 +143,13 @@ where self.inner.write_runtime_handle.spawn(async move { if force || this.pick(entry.key()) { let mut buffer = IoBytesMut::new(); - match EntrySerializer::serialize(entry.key(), entry.value(), &compression, &mut buffer) { + match EntrySerializer::serialize( + entry.key(), + entry.value(), + &compression, + &mut buffer, + &this.inner.metrics, + ) { Ok(info) => { let buffer = buffer.freeze(); this.inner.engine.enqueue(entry, buffer, info); diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs index 8275f710..26132575 100644 --- a/foyer-storage/src/test_utils.rs +++ b/foyer-storage/src/test_utils.rs @@ -14,9 +14,16 @@ //! Test utils for the `foyer-storage` crate. -use std::{borrow::Borrow, collections::HashSet, fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc}; +use std::{ + borrow::Borrow, + collections::HashSet, + fmt::Debug, + hash::Hash, + marker::PhantomData, + sync::{Arc, OnceLock}, +}; -use foyer_common::code::StorageKey; +use foyer_common::{code::StorageKey, metrics::Metrics}; use parking_lot::Mutex; use crate::{ @@ -24,6 +31,14 @@ use crate::{ statistics::Statistics, }; +/// A phantom metrics for test. +static METRICS_FOR_TEST: OnceLock = OnceLock::new(); + +/// Get a phantom metrics for test. +pub fn metrics_for_test() -> &'static Metrics { + METRICS_FOR_TEST.get_or_init(|| Metrics::new("test")) +} + /// A picker that only admits key from the given list. pub struct BiasedPicker { admits: HashSet,