diff --git a/packages/qvac-lib-registry-server/docs/DEPLOYMENT_GUIDE.md b/packages/qvac-lib-registry-server/docs/DEPLOYMENT_GUIDE.md index ca92e20540..0cd0225e7b 100644 --- a/packages/qvac-lib-registry-server/docs/DEPLOYMENT_GUIDE.md +++ b/packages/qvac-lib-registry-server/docs/DEPLOYMENT_GUIDE.md @@ -564,7 +564,8 @@ node scripts/bin.js run --storage ./corestore --metrics-port 0 | `qvac_registry_blob_core_count` | Gauge | Number of blob cores opened locally on this node | | `qvac_registry_blob_core_peers` | Gauge | Peers connected to this node's local blob core (may be partial replicas) | | `qvac_registry_blob_core_seeders` | Gauge | Peers holding this node's local blob core fully and uploading (full replicas) | -| `qvac_registry_blob_core_fully_downloaded` | Gauge | Whether this node's local blob core is fully replicated (1/0) | +| `qvac_registry_blob_core_length` | Gauge | This node's local blob core length in blocks | +| `qvac_registry_blob_core_contiguous_length` | Gauge | Blob core contiguous length in blocks (gap indicates missing blocks on disk) | | `qvac_registry_blob_core_byte_length` | Gauge | Byte length of this node's local blob core | | `qvac_registry_view_core_length` | Gauge | View core length (total blocks) | | `qvac_registry_view_core_contiguous_length` | Gauge | View core contiguous length (gap indicates replication lag) | @@ -572,8 +573,6 @@ node scripts/bin.js run --storage ./corestore --metrics-port 0 | `qvac_registry_rpc_requests_total` | Counter | RPC requests by method | | `qvac_registry_rpc_errors_total` | Counter | RPC errors by method | | `qvac_registry_is_indexer` | Gauge | Whether this node is an indexer | -| `qvac_registry_blind_peers_connected` | Gauge | Number of configured blind peers with an active connection | -| `qvac_registry_blind_peer_connected` | Gauge | Per-blind-peer connection status (labeled by `peer_key`) | `qvac_registry_total_blob_bytes` is derived from the view, not from the on-disk blob cores, so it reports the logical registry size consistently on every node (indexers that do not store blobs locally still report the same value). diff --git a/packages/qvac-lib-registry-server/docs/grafana/REGISTRY_DASHBOARD.json b/packages/qvac-lib-registry-server/docs/grafana/REGISTRY_DASHBOARD.json index 3030064db9..eb766a7b57 100644 --- a/packages/qvac-lib-registry-server/docs/grafana/REGISTRY_DASHBOARD.json +++ b/packages/qvac-lib-registry-server/docs/grafana/REGISTRY_DASHBOARD.json @@ -1142,64 +1142,6 @@ "title": "Indexer", "type": "stat" }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": 0 - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 37 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.0", - "targets": [ - { - "expr": "qvac_registry_blind_peers_connected", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Blind Peers", - "type": "stat" - }, { "datasource": { "type": "prometheus" @@ -1222,7 +1164,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 12, + "x": 8, "y": 37 }, "id": 19, @@ -1294,7 +1236,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 16, + "x": 12, "y": 37 }, "id": 20, @@ -1318,12 +1260,12 @@ "pluginVersion": "12.4.0", "targets": [ { - "expr": "min(qvac_registry_blob_core_fully_downloaded)", + "expr": "min(qvac_registry_blob_core_contiguous_length == bool qvac_registry_blob_core_length) and min(qvac_registry_blob_core_length) > bool 0", "legendFormat": "", "refId": "A" } ], - "title": "Blobs Synced", + "title": "Blob Core Contiguous", "type": "stat" }, { @@ -1349,7 +1291,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 20, + "x": 16, "y": 37 }, "id": 21, diff --git a/packages/qvac-lib-registry-server/lib/metrics.js b/packages/qvac-lib-registry-server/lib/metrics.js index f041eab69e..f0411cc235 100644 --- a/packages/qvac-lib-registry-server/lib/metrics.js +++ b/packages/qvac-lib-registry-server/lib/metrics.js @@ -97,12 +97,18 @@ class QvacMetrics { }) registerGauge({ - name: 'qvac_registry_blob_core_fully_downloaded', - help: 'Whether this node\'s local blob core is fully downloaded (1=yes, 0=no)', + name: 'qvac_registry_blob_core_length', + help: 'This node\'s local blob core length (total blocks)', collect () { - const core = firstBlobCore(self._service) - if (!core || core.length === 0) { this.set(0); return } - this.set(core.contiguousLength === core.length ? 1 : 0) + this.set(firstBlobCore(self._service)?.length ?? 0) + } + }) + + registerGauge({ + name: 'qvac_registry_blob_core_contiguous_length', + help: 'This node\'s local blob core contiguous length (gap = length - contiguous indicates missing blocks on disk)', + collect () { + this.set(firstBlobCore(self._service)?.contiguousLength ?? 0) } }) @@ -140,29 +146,6 @@ class QvacMetrics { } }) - registerGauge({ - name: 'qvac_registry_blind_peers_connected', - help: 'Number of configured blind peers with an active connection', - collect () { - this.set(self._service.getConnectedBlindPeerKeys().length) - } - }) - - registerGauge({ - name: 'qvac_registry_blind_peer_connected', - help: 'Whether each configured blind peer currently has an active connection (1=yes, 0=no)', - labelNames: ['peer_key'], - collect () { - this.reset() - for (const peerKey of self._service.getConfiguredBlindPeerKeys()) { - this.set( - { peer_key: peerKey }, - self._service.isBlindPeerConnected(peerKey) ? 1 : 0 - ) - } - } - }) - registerGauge({ name: 'qvac_registry_blob_core_byte_length', help: 'Byte length of this node\'s local blob core (only populated on nodes that opened the blob core locally)', diff --git a/packages/qvac-lib-registry-server/tests/integration/metrics.integration.test.js b/packages/qvac-lib-registry-server/tests/integration/metrics.integration.test.js index 9c4f8c6025..7be99521be 100644 --- a/packages/qvac-lib-registry-server/tests/integration/metrics.integration.test.js +++ b/packages/qvac-lib-registry-server/tests/integration/metrics.integration.test.js @@ -4,7 +4,6 @@ const test = require('brittle') const Corestore = require('corestore') const Hyperswarm = require('hyperswarm') const http = require('http') -const IdEnc = require('hypercore-id-encoding') const promClient = require('prom-client') const RegistryService = require('../../lib/registry-service') @@ -116,12 +115,12 @@ test('/metrics includes QVAC custom gauges', async (t) => { t.ok(body.includes('qvac_registry_totals_refreshed_age_seconds'), 'has totals_refreshed_age_seconds') t.ok(body.includes('qvac_registry_blob_core_count'), 'has blob_core_count') t.ok(body.includes('qvac_registry_blob_core_seeders'), 'has blob_core_seeders') + t.ok(body.includes('qvac_registry_blob_core_length'), 'has blob_core_length') + t.ok(body.includes('qvac_registry_blob_core_contiguous_length'), 'has blob_core_contiguous_length') t.ok(body.includes('qvac_registry_view_core_length'), 'has view_core_length') t.ok(body.includes('qvac_registry_view_core_contiguous_length'), 'has view_core_contiguous_length') t.ok(body.includes('qvac_registry_view_core_seeders'), 'has view_core_seeders') t.ok(body.includes('qvac_registry_is_indexer'), 'has is_indexer') - t.ok(body.includes('qvac_registry_blind_peers_connected'), 'has blind_peers_connected') - t.ok(body.includes('qvac_registry_blind_peer_connected'), 'has blind_peer_connected') t.ok(body.includes('qvac_registry_rpc_requests_total'), 'has rpc_requests_total') t.ok(body.includes('qvac_registry_rpc_errors_total'), 'has rpc_errors_total') @@ -135,10 +134,6 @@ test('/metrics includes QVAC custom gauges', async (t) => { t.ok(modelCountLine, 'exports model_count as a single series') t.ok(modelCountLine.endsWith(' 0'), 'model_count is 0 on an empty registry') - t.absent(body.includes('qvac_registry_models_total'), 'legacy models_total name is removed') - t.absent(body.includes('qvac_registry_blob_cores_total'), 'legacy blob_cores_total name is removed') - t.absent(body.includes('qvac_registry_model_size_bytes'), 'per-path model_size_bytes metric is removed') - const viewSeedersLine = body.split('\n') .find(line => line.startsWith('qvac_registry_view_core_seeders ')) t.ok(viewSeedersLine, 'exports view_core_seeders as a single series') @@ -182,50 +177,6 @@ test('RPC metrics counters increment', async (t) => { } }) -test('blind peer metrics track configured peers with active connections', async (t) => { - const blindPeerKeys = [ - IdEnc.normalize(Buffer.alloc(32, 1)), - IdEnc.normalize(Buffer.alloc(32, 2)) - ] - const ctx = await createServiceWithMetrics(t) - - try { - ctx.service.blindPeerKeys = blindPeerKeys - ctx.service._trackPeerConnection(blindPeerKeys[0]) - ctx.service._trackPeerConnection(blindPeerKeys[0]) - ctx.service._trackPeerConnection('writer-peer') - - let res = await httpGet(ctx.port, '/metrics') - let body = res.body - - const connectedPeerLine = body.split('\n') - .find(line => line.startsWith(`qvac_registry_blind_peer_connected{peer_key="${blindPeerKeys[0]}"}`)) - t.ok(connectedPeerLine, 'has connected blind peer series') - t.ok(connectedPeerLine.endsWith(' 1'), 'connected blind peer is reported as 1') - - const disconnectedPeerLine = body.split('\n') - .find(line => line.startsWith(`qvac_registry_blind_peer_connected{peer_key="${blindPeerKeys[1]}"}`)) - t.ok(disconnectedPeerLine, 'has disconnected blind peer series') - t.ok(disconnectedPeerLine.endsWith(' 0'), 'disconnected blind peer is reported as 0') - - ctx.service._untrackPeerConnection(blindPeerKeys[0]) - ctx.service._untrackPeerConnection(blindPeerKeys[0]) - - res = await httpGet(ctx.port, '/metrics') - body = res.body - - const afterCloseCountLine = body.split('\n') - .find(line => line.startsWith('qvac_registry_blind_peers_connected ')) - t.ok(afterCloseCountLine.endsWith(' 0'), 'blind peer count drops after connection closes') - - const afterClosePeerLine = body.split('\n') - .find(line => line.startsWith(`qvac_registry_blind_peer_connected{peer_key="${blindPeerKeys[0]}"}`)) - t.ok(afterClosePeerLine.endsWith(' 0'), 'blind peer status drops after connection closes') - } finally { - await cleanup(ctx) - } -}) - test('MetricsServer closes cleanly', async (t) => { const ctx = await createServiceWithMetrics(t)