Skip to content

feat: async block import persistence #8784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
twoeths merged 28 commits into from
Jan 30, 2026
Merged

feat: async block import persistence #8784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
8744f60
feat: async block import persistence
wemeetagain Jan 23, 2026
58cc8fb
chore: remove eagerPersistBlock
wemeetagain Jan 23, 2026
16f0733
chore: improve comments
wemeetagain Jan 23, 2026
a1df1d9
chore: simplify
wemeetagain Jan 23, 2026
958cbdc
chore: add support for serving pending blob sidecars
wemeetagain Jan 23, 2026
2ecf716
chore: add utility get sidecar chain methods
wemeetagain Jan 23, 2026
12c1cf4
chore: use chain methods to serve sidecars
wemeetagain Jan 23, 2026
05b0403
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 23, 2026
d5931b0
chore: add comment
wemeetagain Jan 23, 2026
0176e92
chore: getBlobSidecars only return all blobs
wemeetagain Jan 23, 2026
9f89be2
chore: lint fix
wemeetagain Jan 23, 2026
42ab4a2
chore: don't await persist job in importBlock
wemeetagain Jan 27, 2026
e1b58ba
chore: add unfinalized block writes queue metrics
wemeetagain Jan 27, 2026
612a281
chore: add error log for persistBlockInputs
wemeetagain Jan 27, 2026
691afda
chore: add block slot and root to error messages
wemeetagain Jan 27, 2026
49959c9
chore: use serialized cache to get serialized column sidecars
wemeetagain Jan 27, 2026
9a843af
chore: more work on reqresp handlers
wemeetagain Jan 27, 2026
45cf2f2
chore: add one more comment
wemeetagain Jan 27, 2026
0faea9a
chore: remove block input prune-on-success
wemeetagain Jan 27, 2026
59d7a30
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 27, 2026
e16cb91
chore: fix download_dashboards script (install its dependency)
wemeetagain Jan 27, 2026
83952f0
chore: add dashboard panels for new metrics
wemeetagain Jan 27, 2026
f2900c5
chore: graceful shutdown
wemeetagain Jan 27, 2026
5125497
Apply suggestions from code review
wemeetagain Jan 28, 2026
15aef31
chore: add better logs
wemeetagain Jan 28, 2026
23aac57
chore: better error messages
wemeetagain Jan 29, 2026
f359c4f
chore: simplify data column sidecar getters interfaces
wemeetagain Jan 29, 2026
5c6e69e
Merge branch 'unstable' into cayman/unfinalized-block-write-queue
wemeetagain Jan 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
337 changes: 337 additions & 0 deletions dashboards/lodestar_beacon_chain.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1326,6 +1326,343 @@
],
"title": "Grow",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 60
},
"id": 39,
"panels": [],
"title": "Unfinalized Block Writes Queue",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 61
},
"id": 40,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "lodestar_unfinalized_block_writes_queue_length",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Queue Length",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 61
},
"id": 41,
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Magma",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false
}
},
"pluginVersion": "10.4.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(lodestar_unfinalized_block_writes_queue_job_time_seconds_bucket[$rate_interval])",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",
"range": true,
"refId": "A"
}
],
"title": "Job Time",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 69
},
"id": 43,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "lodestar_unfinalized_block_writes_queue_concurrency",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Queue Concurrency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"id": 42,
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Magma",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false
}
},
"pluginVersion": "10.4.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(lodestar_unfinalized_block_writes_queue_job_wait_time_seconds_bucket[$rate_interval])",
"format": "heatmap",
"instant": false,
"legendFormat": "{{le}}",
"range": true,
"refId": "A"
}
],
"title": "Job Wait Time",
"type": "heatmap"
}
],
"refresh": "10s",
Expand Down
Loading
Loading